CCDB: handle access to CCDB via security proxy (#15724)

Gate tokens are minted per broker route, so a process facing both the
writable test CCDB and production needs one token per endpoint.
ALICEO2_CCDB_AUTH_TOKENS ("<url>=<tok>;<url>=<tok>") replaces the single
ALICEO2_CCDB_AUTH_TOKEN, and the token is chosen by the URL each request
targets:

* header lists are built inside the hostsPool loops -- built once
  outside, every host received the first host's token;
* matching stops at a path boundary, so a missing ".../ccdb-prod" entry
  cannot silently fall back to the ".../ccdb" token (an opaque 401);
* CURLOPT_HTTPHEADER is set unconditionally: with per-host lists,
  skipping the set would leave a freed list installed on the handle.

Incoming header values are trimmed before reuse: header_callback stores
raw lines, so the ETag kept its CRLF and, spliced into If-None-Match,
ended the request header block early -- dropping the Authorization
header appended after it (401 instead of 304, testCcdbApi.cxx:461).

DeadChannelMapCreator now checks the fetch before reading Valid-From/
Until, so an unreachable CCDB reports instead of throwing out_of_range.
CalDetStreamerTest reads ALICEO2_CCDB_PRODUCTION_HOST (default
unchanged): its pinned timestamp matches the production object, and the
test instance holds a different object at that path.

Author: ktf

CCDB/include/CCDB/CCDBDownloader.h

@@ -53,7 +53,12 @@ typedef struct DownloaderRequestData {
   HeaderObjectPair_t hoPair;
   std::map<std::string, std::string>* headers;
   std::string userAgent;
-  curl_slist* optionsList;
+  // One header list per entry of `hosts`, parallel to it. Per host and not one
+  // shared list because the gate token a broker expects is per endpoint: a
+  // multi-host pool can mix them, and tryNewHost() swapping only the URL left
+  // the second host receiving the first host's token -- answered 401, so the
+  // failover silently retrieved nothing (testCcdbApi multi_host_test).
+  std::vector<curl_slist*> optionsLists;
 
   std::function<bool(std::string)> localContentCallback;
 } DownloaderRequestData;
@@ -304,7 +309,8 @@ class CCDBDownloader
     int hostInd;
     int locInd;
     DownloaderRequestData* requestData;
-    curl_slist** options;
+    // Freed by transferFinished; indexed by hostInd, see DownloaderRequestData.
+    std::vector<curl_slist*>* options;
   } PerformData;
 #endif
 

CCDB/include/CCDB/CcdbApi.h

@@ -18,6 +18,7 @@
 #define PROJECT_CCDBAPI_H
 
 #include <string>
+#include <string_view>
 #include <memory>
 #include <map>
 #include <curl/curl.h>
@@ -581,7 +582,7 @@ class CcdbApi //: public DatabaseInterface
   void initCurlOptionsForRetrieve(CURL* curlHandle, void* pointer, CurlWriteCallback writeCallback, bool followRedirect = true) const;
 
   /// initialize HTTPS header information for the CURL handle. Needs to be given an existing curl_slist* pointer to work with (may be nullptr), which needs to be free by the caller.
-  void initCurlHTTPHeaderOptionsForRetrieve(CURL* curlHandle, curl_slist*& option_list, long timestamp, std::map<std::string, std::string>* headers, std::string const& etag, const std::string& createdNotAfter, const std::string& createdNotBefore) const;
+  void initCurlHTTPHeaderOptionsForRetrieve(CURL* curlHandle, curl_slist*& option_list, long timestamp, std::map<std::string, std::string>* headers, std::string const& etag, const std::string& createdNotAfter, const std::string& createdNotBefore, std::string_view url) const;
 
   bool receiveToFile(FILE* fileHandle, std::string const& path, std::map<std::string, std::string> const& metadata,
                      long timestamp, std::map<std::string, std::string>* headers = nullptr, std::string const& etag = "",

CCDB/src/CCDBDownloader.cxx

@@ -365,6 +365,14 @@ void CCDBDownloader::tryNewHost(PerformData* performData, CURL* easy_handle)
   LOG(debug) << "Connecting to another host " << newUrl << "\n";
   requestData->hoPair.header.clear();
   curl_easy_setopt(easy_handle, CURLOPT_URL, newUrl.c_str());
+  // The headers travel with the host, not with the request: a broker mints its
+  // gate token per endpoint, so carrying the previous host's list here is what
+  // made the failover arrive unauthenticated. The lists are built per host by
+  // CcdbApi::scheduleDownload, which is where the token table is visible.
+  if (performData->hostInd < static_cast<int>(requestData->optionsLists.size())) {
+    curl_easy_setopt(easy_handle, CURLOPT_HTTPHEADER,
+                     requestData->optionsLists.at(performData->hostInd));
+  }
   mHandlesToBeAdded.push_back(easy_handle);
 }
 
@@ -568,7 +576,9 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
           }
         }
         --(*performData->requestsLeft);
-        curl_slist_free_all(*performData->options);
+        for (auto* optionList : *performData->options) {
+          curl_slist_free_all(optionList);
+        }
         delete requestData;
         delete performData->codeDestination;
         curl_easy_cleanup(easy_handle);
@@ -729,7 +739,7 @@ void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter)
   curl_easy_getinfo(handle, CURLINFO_PRIVATE, &requestData);
   headerMap = &(requestData->hoPair.header);
   hostsPool = &(requestData->hosts);
-  auto* options = &(requestData->optionsList);
+  auto* options = &(requestData->optionsLists);
 
   // Prepare temporary data about transfer
   auto* data = new CCDBDownloader::PerformData(); // Freed in transferFinished