From 5ac588db4784cc4abef6e605b31277a0422f377f Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 2 Sep 2026 17:23:38 +0200 Subject: [PATCH] feat (FileStorage): allow to specify block size --- src/DIRAC/Resources/Storage/FileStorage.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/DIRAC/Resources/Storage/FileStorage.py b/src/DIRAC/Resources/Storage/FileStorage.py index 4f91040155a..c42feab2ac4 100644 --- a/src/DIRAC/Resources/Storage/FileStorage.py +++ b/src/DIRAC/Resources/Storage/FileStorage.py @@ -148,6 +148,20 @@ def __init__(self, storageName, parameters): self.pluginName = "File" self.protocol = self.protocolParameters["Protocol"] + self.block_size = int(self._allProtocolParameters.get("BlockSize", 0)) + + def _do_copy(self, src, dst): + """ + Either use shutil.copy2 or copyfileobj if the site is + strongly opinionated on the block size + """ + if not self.block_size: + return shutil.copy2(src, dst) + else: + with open(src, "rb") as fd_r: + with open(dst, "wb") as fd_w: + shutil.copyfileobj(fd_r, fd_w, self.block_size) + shutil.copystat(src, dst) def getURLBase(self, withWSUrl=False): return S_OK(self.basePath) @@ -243,7 +257,7 @@ def getFile(self, path, localPath=False): try: fileName = os.path.basename(src_url) dest_url = os.path.join(localPath, fileName) - shutil.copy2(src_url, dest_url) + self._do_copy(src_url, dest_url) fileSize = os.path.getsize(dest_url) successful[src_url] = fileSize @@ -277,7 +291,7 @@ def putFile(self, path, sourceSize=0): dirname = os.path.dirname(dest_url) if not os.path.exists(dirname): os.makedirs(dirname) - shutil.copy2(src_file, dest_url) + self._do_copy(src_file, dest_url) fileSize = os.path.getsize(dest_url) try: src_cks = fileAdler(src_file)