Skip to content

[int] Parameterise SQL in DMS FileMetadata #8646

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sfayer wants to merge 1 commit into
base: integration
Choose a base branch
from
Open

[int] Parameterise SQL in DMS FileMetadata #8646

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 72 additions & 29 deletions src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ def addMetadataField(self, pName, pType, credDict):
if pName in FILE_STANDARD_METAKEYS:
return S_ERROR("Illegal use of reserved metafield name")

result = self.db._checkIdentifier(pName)
if not result["OK"]:
return result

result = self.db.dmeta._getMetadataFields(credDict)
if not result["OK"]:
return result
Expand All @@ -56,10 +60,13 @@ def addMetadataField(self, pName, pType, credDict):
valueType = pType
if pType == "MetaSet":
valueType = "VARCHAR(64)"
req = (
"CREATE TABLE FC_FileMeta_%s ( FileID INTEGER NOT NULL, Value %s, PRIMARY KEY (FileID), INDEX (Value) )"
% (pName, valueType)
)
result = self.db._checkType(valueType)
if not result["OK"]:
return result
valueType = result["Value"]
req = "CREATE TABLE "
req += f"FC_FileMeta_{pName} "
req += f"(FileID INTEGER NOT NULL, Value {valueType}, PRIMARY KEY (FileID), INDEX (Value))"
result = self.db._query(req)
if not result["OK"]:
return result
Expand All @@ -83,13 +90,17 @@ def deleteMetadataField(self, pName, credDict):
:return: S_OK/S_ERROR
"""

result = self.db._checkIdentifier(pName)
if not result["OK"]:
return result

req = f"DROP TABLE FC_FileMeta_{pName}"
result = self.db._update(req)
error = ""
if not result["OK"]:
error = result["Message"]
req = f"DELETE FROM FC_FileMetaFields WHERE MetaName='{pName}'"
result = self.db._update(req)
req = "DELETE FROM FC_FileMetaFields WHERE MetaName=%s"
result = self.db._update(req, args=(pName,))
if not result["OK"]:
if error:
result["Message"] = error + "; " + result["Message"]
Expand Down Expand Up @@ -158,11 +169,17 @@ def setMetadata(self, path, metaDict, credDict):
return S_ERROR(f"Field {metaName} not indexed, but ForceIndexedMetadata is set", callStack=[])
result = self.__setFileMetaParameter(fileID, metaName, metaValue, credDict)
else:
result = self.db._checkIdentifier(metaName)
if not result["OK"]:
return result
result = self.db.insertFields(f"FC_FileMeta_{metaName}", ["FileID", "Value"], [fileID, metaValue])
if not result["OK"]:
if result["Message"].find("Duplicate") != -1:
req = "UPDATE FC_FileMeta_%s SET Value='%s' WHERE FileID=%d" % (metaName, metaValue, fileID)
result = self.db._update(req)
req = "UPDATE "
req += f"FC_FileMeta_{metaName} "
req += "SET Value=%s WHERE FileID=%s"
args = (metaValue, str(fileID))
result = self.db._update(req, args=args)
if not result["OK"]:
return result
else:
Expand Down Expand Up @@ -196,14 +213,21 @@ def removeMetadata(self, path, metadata, credDict):
for meta in metadata:
if meta in metaFields:
# Indexed meta case
req = "DELETE FROM FC_FileMeta_%s WHERE FileID=%d" % (meta, fileID)
result = self.db._update(req)
result = self.db._checkIdentifier(meta)
if not result["OK"]:
failedMeta[meta] = result["Message"]
continue
req = "DELETE FROM "
req += f"FC_FileMeta_{meta} "
req += "WHERE FileID=%s"
result = self.db._update(req, args=(str(fileID),))
if not result["OK"]:
failedMeta[meta] = result["Value"]
else:
# Meta parameter case
req = "DELETE FROM FC_FileMeta WHERE MetaKey='%s' AND FileID=%d" % (meta, fileID)
result = self.db._update(req)
req = "DELETE FROM FC_FileMeta WHERE MetaKey=%s AND FileID=%s"
args = (meta, str(fileID))
result = self.db._update(req, args=args)
if not result["OK"]:
failedMeta[meta] = result["Value"]

Expand Down Expand Up @@ -280,19 +304,25 @@ def _getFileUserMetadataByID(self, fileIDList, credDict, connection=False):
return result
metaFields = result["Value"]

stringIDs = ",".join([f"{fId}" for fId in fileIDList])
metaDict = {}
placeholders = ",".join(["%s"] * len(fileIDList))
for meta in metaFields:
req = f"SELECT Value,FileID FROM FC_FileMeta_{meta} WHERE FileID in ({stringIDs})"
result = self.db._query(req, conn=connection)
req = "SELECT Value,FileID FROM "
req += f"FC_FileMeta_{meta} "
req += "WHERE FileID in ("
req += placeholders
req += ")"
result = self.db._query(req, args=fileIDList, conn=connection)
if not result["OK"]:
return result
for value, fileID in result["Value"]:
metaDict.setdefault(fileID, {})
metaDict[fileID][meta] = value

req = f"SELECT FileID,MetaKey,MetaValue from FC_FileMeta where FileID in ({stringIDs})"
result = self.db._query(req, conn=connection)
req = "SELECT FileID,MetaKey,MetaValue FROM FC_FileMeta WHERE FileID IN ("
req += placeholders
req += ")"
result = self.db._query(req, args=fileIDList, conn=connection)
if not result["OK"]:
return result
for fileID, key, value in result["Value"]:
Expand Down Expand Up @@ -323,8 +353,10 @@ def getFileUserMetadata(self, path, credDict):
metaDict = {}
metaTypeDict = {}
for meta in metaFields:
req = "SELECT Value,FileID FROM FC_FileMeta_%s WHERE FileID=%d" % (meta, fileID)
result = self.db._query(req)
req = "SELECT Value,FileID FROM "
req += f"FC_FileMeta_{meta} "
req += "WHERE FileID=%s"
result = self.db._query(req, args=(str(fileID),))
if not result["OK"]:
return result
if result["Value"]:
Expand All @@ -350,8 +382,8 @@ def __getFileMetaParameters(self, fileID, credDict):
:return: S_OK/S_ERROR, Value - dict of meta parameters
"""

req = "SELECT FileID,MetaKey,MetaValue from FC_FileMeta where FileID=%d " % fileID
result = self.db._query(req)
req = "SELECT FileID,MetaKey,MetaValue from FC_FileMeta where FileID=%s"
result = self.db._query(req, args=(str(fileID),))
if not result["OK"]:
return result
if not result["Value"]:
Expand Down Expand Up @@ -390,24 +422,29 @@ def __transformMetaParameterToData(self, metaName):
:return: S_OK/S_ERROR
"""

req = f"SELECT FileID,MetaValue from FC_FileMeta WHERE MetaKey='{metaName}'"
result = self.db._query(req)
req = "SELECT FileID,MetaValue from FC_FileMeta WHERE MetaKey=%s"
result = self.db._query(req, args=(metaName,))
if not result["OK"]:
return result
if not result["Value"]:
return S_OK()

insertValueList = []
for fileID, meta in result["Value"]:
insertValueList.append("( %d,'%s' )" % (fileID, meta))
insertValueList.append((str(fileID), meta))

req = f"INSERT INTO FC_FileMeta_{metaName} (FileID,Value) VALUES {', '.join(insertValueList)}"
result = self.db._update(req)
result = self.db._checkIdentifier(metaName)
if not result["OK"]:
return result
req = "INSERT INTO "
req += f"FC_FileMeta_{metaName} "
req += "(FileID,Value) VALUES (%s,%s)"
result = self.db._updatemany(req, data=insertValueList)
if not result["OK"]:
return result

req = f"DELETE FROM FC_FileMeta WHERE MetaKey='{metaName}'"
result = self.db._update(req)
req = "DELETE FROM FC_FileMeta WHERE MetaKey=%s"
result = self.db._update(req, args=(metaName,))
return result

#########################################################################
Expand Down Expand Up @@ -507,7 +544,8 @@ def __buildSEQuery(self, storageElements):
seID = self.db.seNames.get(se, -1)
if seID == -1:
return S_ERROR(f"Unknown SE {se}")
seIDList.append(seID)
# Explicit int cast to prevent injection via SE lookup table
seIDList.append(int(seID))
table = "FC_Replicas"
seString = intListToString(seIDList)
query = "%%s.SEID IN ( %s )" % seString
Expand All @@ -525,6 +563,9 @@ def __buildUserMetaQuery(self, userMetaDict):
resultList = []
leftJoinTables = []
for meta, value in userMetaDict.items():
result = self.db._checkIdentifier(meta)
if not result["OK"]:
return result
table = f"FC_FileMeta_{meta}"

result = self.__createMetaSelection(value)
Expand Down Expand Up @@ -644,6 +685,8 @@ def __findFilesByMetadata(self, metaDict, dirList, credDict):
tables = []

if dirList:
# Ensure all dir IDs are integers to prevent injection via string interpolation
dirList = [int(d) for d in dirList]
dirString = intListToString(dirList)
conditions.append(f"F.DirID in ({dirString})")

Expand Down
Loading