From 563a095ef0a99866a61822f48be5a16b1aaca35b Mon Sep 17 00:00:00 2001 From: Maximilian Linhoff Date: Fri, 21 Aug 2026 12:53:46 +0200 Subject: [PATCH 1/5] refactor: add helper method to obtain rucio did type --- .../Catalog/RucioFileCatalogClient.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py index 895ecc65809..75b5912b062 100644 --- a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py +++ b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py @@ -248,6 +248,10 @@ def getReplicas(self, lfns, allStatus=False): return S_ERROR(str(err)) return result + def _get_did_type(self, scope, name): + """Return type of rucio DID (CONTAINER, DATASET or FILE).""" + return self.client.get_did(scope, name)["type"] + @checkCatalogArguments def listDirectory(self, lfns, verbose=False): """ @@ -263,8 +267,8 @@ def listDirectory(self, lfns, verbose=False): try: did = self.__getDidsFromLfn(lfn) # First need to check if it's a dataset or container - meta = self.client.get_metadata(did["scope"], did["name"]) - if meta["did_type"] == "CONTAINER": + did_type = self._get_did_type(did["scope"], did["name"]) + if did_type == "CONTAINER": if lfn not in result["Value"]["Successful"]: result["Value"]["Successful"][lfn] = {"Files": {}, "Links": {}, "SubDirs": {}} for child in self.client.list_content(scope=did["scope"], name=did["name"]): @@ -292,7 +296,7 @@ def listDirectory(self, lfns, verbose=False): result["Value"]["Successful"][lfn]["Files"][childName] = {"Mode": 509} if verbose: pass - elif meta["did_type"] == "DATASET": + elif did_type == "DATASET": file_dict = {} for file_did in self.client.list_files(scope=did["scope"], name=did["name"]): guid = file_did["guid"] @@ -612,8 +616,8 @@ def removeFile(self, lfns): for lfn in lfns: try: did = self.__getDidsFromLfn(lfn) - meta = self.client.get_metadata(did["scope"], did["name"]) - if meta["did_type"] == "FILE": + did_type = self._get_did_type(did["scope"], did["name"]) + if did_type == "FILE": parentLfn = "/".join(lfn.split("/")[:-1]) parentDid = self.__getDidsFromLfn(parentLfn) dsnScope, dsnName = parentDid["scope"], parentDid["name"] @@ -637,8 +641,8 @@ def removeDirectory(self, lfns): for lfn in lfns: try: did = self.__getDidsFromLfn(lfn) - meta = self.client.get_metadata(did["scope"], did["name"]) - if meta["did_type"] == "DATASET": + did_type = self._get_did_type(did["scope"], did["name"]) + if did_type == "DATASET": try: self.client.set_metadata(scope=did["scope"], name=did["name"], key="lifetime", value=1) resDict["Successful"][lfn] = True From d60c3fd53484c2eab9909278fc1460e5a8735e0a Mon Sep 17 00:00:00 2001 From: Maximilian Linhoff Date: Fri, 21 Aug 2026 13:30:30 +0200 Subject: [PATCH 2/5] refactor: Improve readability in RucioFileCatalogClient --- .../Resources/Catalog/RucioFileCatalogClient.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py index 75b5912b062..95ce0871275 100644 --- a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py +++ b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py @@ -24,6 +24,8 @@ sLog = gLogger.getSubLogger(__name__) +RUCIO_COLLECTION_TYPES = {"DATASET", "CONTAINER"} + def get_scope(lfn, scopes=None, diracAlgorithm="dirac"): """ @@ -305,11 +307,7 @@ def listDirectory(self, lfns, verbose=False): file_dict[file_did["name"]] = str(uuid.UUID(guid)) if lfn not in result["Value"]["Successful"]: result["Value"]["Successful"][lfn] = {"Files": {}, "Links": {}, "SubDirs": {}} - for rep in self.client.list_replicas( - [ - did, - ] - ): + for rep in self.client.list_replicas([did]): if rep: name = rep["name"] if self.convertUnicode: @@ -343,7 +341,7 @@ def listDirectory(self, lfns, verbose=False): "PFN": pfn, "Status": "U", } - except DataIdentifierNotFound as err: + except DataIdentifierNotFound: result["Value"]["Failed"][lfn] = "No such file or directory" except Exception as err: return S_ERROR(str(err)) @@ -360,14 +358,14 @@ def getFileMetadata(self, lfns, ownership=False): dids = [self.__getDidsFromLfn(lfn) for lfn in chunk] for meta in self.client.get_metadata_bulk(dids): lfn = str(meta["name"]) - if meta["did_type"] in ["DATASET", "CONTAINER"]: + if meta["did_type"] in RUCIO_COLLECTION_TYPES: nlinks = len([child for child in self.client.list_content(meta["scope"], meta["name"])]) successful[lfn] = { "Checksum": "", "ChecksumType": "", "CreationDate": meta["created_at"], "GUID": "", - "Mode": 509, + "Mode": 0o775, "ModificationDate": meta["updated_at"], "NumberOfLinks": nlinks, "Size": 0, @@ -386,7 +384,7 @@ def getFileMetadata(self, lfns, ownership=False): "ChecksumType": "AD", "CreationDate": meta["created_at"], "GUID": guid, - "Mode": 436, + "Mode": 0o664, "ModificationDate": meta["updated_at"], "NumberOfLinks": 1, "Size": meta["bytes"], From d6b55b43a9f72269aa23ffdd9789ae25033286dc Mon Sep 17 00:00:00 2001 From: Maximilian Linhoff Date: Fri, 21 Aug 2026 13:58:35 +0200 Subject: [PATCH 3/5] fix: compatibility of older rucio clients with rucio 41 server --- .../Catalog/RucioFileCatalogClient.py | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py index 95ce0871275..eb34d8bcddf 100644 --- a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py +++ b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py @@ -356,7 +356,7 @@ def getFileMetadata(self, lfns, ownership=False): for chunk in lfnChunks: try: dids = [self.__getDidsFromLfn(lfn) for lfn in chunk] - for meta in self.client.get_metadata_bulk(dids): + for meta in self.client.get_metadata_bulk(dids, plugin="DID_COLUMN"): lfn = str(meta["name"]) if meta["did_type"] in RUCIO_COLLECTION_TYPES: nlinks = len([child for child in self.client.list_content(meta["scope"], meta["name"])]) @@ -410,8 +410,8 @@ def exists(self, lfns): for lfn in lfns: try: did = self.__getDidsFromLfn(lfn) + self.client.get_did(did["scope"], did["name"]) exists = True - self.client.get_metadata(did["scope"], did["name"]) except DataIdentifierNotFound: exists = False except Exception as err: @@ -426,8 +426,8 @@ def getFileSize(self, lfns): for lfn in lfns: try: did = self.__getDidsFromLfn(lfn) - meta = self.client.get_metadata(did["scope"], did["name"]) - if meta["did_type"] == "FILE": + meta = self.client.get_did(did["scope"], did["name"]) + if meta["type"] == "FILE": result["Value"]["Successful"][lfn] = meta["bytes"] else: result["Value"]["Successful"][lfn] = 0 @@ -443,9 +443,9 @@ def isDirectory(self, lfns): result = {"Successful": {}, "Failed": {}} dids = [self.__getDidsFromLfn(lfn) for lfn in lfns] try: - for meta in self.client.get_metadata_bulk(dids): + for meta in self.client.get_metadata_bulk(dids, plugin="DID_COLUMN"): lfn = str(meta["name"]) - result["Successful"][lfn] = meta["did_type"] in ["DATASET", "CONTAINER"] + result["Successful"][lfn] = meta["did_type"] in RUCIO_COLLECTION_TYPES for lfn in lfns: if lfn not in result["Successful"] and lfn not in result["Failed"]: result["Failed"][lfn] = "No such file or directory" @@ -461,9 +461,9 @@ def isFile(self, lfns): for lfn in lfns: dids.append(self.__getDidsFromLfn(lfn)) try: - for meta in self.client.get_metadata_bulk(dids): + for meta in self.client.get_metadata_bulk(dids, plugin="DID_COLUMN"): lfn = str(meta["name"]) - result["Successful"][lfn] = meta["did_type"] in ["FILE"] + result["Successful"][lfn] = meta["did_type"] == "FILE" for lfn in lfns: if lfn not in result["Successful"] and lfn not in result["Failed"]: result["Failed"][lfn] = "No such file or directory" @@ -535,7 +535,7 @@ def addReplica(self, lfns): try: did = self.__getDidsFromLfn(lfn) if not size or not checksum: - meta = self.client.get_metadata(did["scope"], did["name"]) + meta = self.client.get_did(did["scope"], did["name"]) size = meta["bytes"] checksum = meta["adler32"] rep = {"scope": did["scope"], "name": did["name"], "bytes": size, "adler32": checksum} @@ -577,14 +577,14 @@ def removeReplica(self, lfns): se = info["SE"] try: did = self.__getDidsFromLfn(lfn) - meta = self.client.get_metadata(did["scope"], did["name"]) - if meta["did_type"] == "FILE": + did_type = self._get_did_type(did["scope"], did["name"]) + if did_type == "FILE": # For file cannot use dataset_locks to identify the rule for rule in self.client.list_did_rules(did["scope"], did["name"]): rid = rule["id"] self.client.update_replication_rule(rid, options={"lifetime": -86400}) successful[lfn] = True - elif meta["did_type"] == "DATASET": + elif did_type == "DATASET": rules = {} for lock in self.client.get_dataset_locks(did["scope"], did["name"]): rule_id = lock["rule_id"] @@ -661,11 +661,11 @@ def getDirectorySize(self, lfns, longOutput=False, rawFiles=False): for lfn in lfns: try: did = self.__getDidsFromLfn(lfn) - meta = self.client.get_metadata(did["scope"], did["name"]) - if meta["did_type"] == "FILE": + did_type = self._get_did_type(did["scope"], did["name"]) + if did_type == "FILE": resDict["Failed"][lfn] = "Not a directory" - elif meta["did_type"] == "CONTAINER": + elif did_type == "CONTAINER": resDict["Successful"][lfn] = { "ClosedDirs": [], "Files": 0, @@ -730,7 +730,6 @@ def getFileUserMetadataBulk(self, lfns): all their parents """ resDict = {"Successful": {}, "Failed": {}} - dids = [] lfnChunks = breakListIntoChunks(lfns, 1000) for lfnList in lfnChunks: try: From 48bf0195905dacf35cd54a4775368943341c4f15 Mon Sep 17 00:00:00 2001 From: Maximilian Linhoff Date: Fri, 21 Aug 2026 13:59:05 +0200 Subject: [PATCH 4/5] fix: Error handling of chunked requests in RucioFileCatalogClient --- src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py index eb34d8bcddf..6693f64bfa3 100644 --- a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py +++ b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py @@ -395,7 +395,8 @@ def getFileMetadata(self, lfns, ownership=False): except ValueError: pass except DataIdentifierNotFound as err: - failed[lfn] = str(err) + for lfn in chunk: + failed[lfn] = str(err) except Exception as err: return S_ERROR(str(err)) for lfn in listFiles: From 9d67c7a0435957d8b997c0f0e77547532ba1af99 Mon Sep 17 00:00:00 2001 From: Maximilian Linhoff Date: Tue, 25 Aug 2026 18:07:15 +0200 Subject: [PATCH 5/5] Fix: Let RucioFileCatalogClient return error for files in listDirectory --- src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py index 6693f64bfa3..8700a8eb237 100644 --- a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py +++ b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py @@ -341,6 +341,8 @@ def listDirectory(self, lfns, verbose=False): "PFN": pfn, "Status": "U", } + else: + result["Value"]["Failed"][lfn] = "Not a directory" except DataIdentifierNotFound: result["Value"]["Failed"][lfn] = "No such file or directory" except Exception as err: