Release v1.10

.github/workflows/build-libra.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   docker:
     env:
-        IMAGE_TAG: "1.8.1"
+        IMAGE_TAG: "1.10"
     runs-on: ubuntu-latest
     steps:
         - name: Get branch names

.java-version

@@ -0,0 +1 @@
+11

changelog.md

@@ -1,14 +1,27 @@
 # Changelog
 
+## Release 1.10 [Unreleased]
+Breif summary:
 
-## Release 1.9 [unreleased]
+### Breaking changes
+
+### Other changes
+
+---
 
+## Release 1.9 [Released 9/2/2025]
+Brief summary
 - Load Tableau database with biopsy_tracker data
 - Insert dlu_upload_type to dlu_package_inventory table
 - Create recalled packages endpoint
 - Update Multi Modal package name
 - Tweak bulk upload fields
 
+### Breaking changes
+- changed column names in tables
+- added new columns to dlu_package_inventory table
+
+---
 
 ## Release 1.8.1 [Released 11/8/2024]
 Brief summary of what's in this release:

data_management/BulkUploader

@@ -0,0 +1,25 @@
+FROM python:3.10-slim-bullseye
+
+WORKDIR /usr/src/app
+
+ENV FLASK_APP=app.py
+ENV FLASK_RUN_HOST=0.0.0.0
+
+RUN apt-get update \
+    && apt-get install -y curl
+
+COPY requirements.txt ./
+
+RUN pip3 config --user set global.progress_bar off
+RUN pip3 install --no-cache-dir -r requirements.txt
+RUN pip3 install -U flask-cors
+
+COPY lib/ ./lib
+COPY main.py ./
+COPY app.py ./
+COPY process_bulk_uploads.py ./
+COPY services/ ./services
+COPY model/ ./model
+COPY .env ./.env
+
+ENTRYPOINT []

data_management/DluWatcher

@@ -1,5 +1,7 @@
 FROM python:3.10-slim-bullseye
 
+USER root
+
 COPY requirements.txt ./
 
 RUN pip3 install --progress-bar off --no-cache-dir -r requirements.txt
@@ -10,6 +12,7 @@ COPY ./services/dlu_filesystem.py ./services/dlu_filesystem.py
 COPY ./services/dlu_package_inventory.py ./services/dlu_package_inventory.py
 COPY ./services/dlu_state.py ./services/dlu_state.py
 COPY ./services/dlu_management.py ./services/dlu_management.py
+COPY ./services/slide_management.py ./services/slide_management.py
 COPY ./services/dlu_mongo.py ./services/dlu_mongo.py
 COPY ./model ./model
 COPY ./watch_files.py ./

data_management/Dockerfile

@@ -31,5 +31,5 @@ COPY app.py ./
 COPY process_bulk_uploads.py ./
 COPY services/ ./services
 
-ENTRYPOINT ["gunicorn", "-b", ":5000", "app:app"]
+ENTRYPOINT ["gunicorn", "-b", ":5000", "app:app", "-t", "1200"]
 

data_management/app.py

@@ -82,7 +82,7 @@ def recall_dlu_package(package_id):
         return error_msg
 
     dlu_data_directory = '/data/package_' + package_id
-    directory_info = DirectoryInfo(dlu_data_directory)
+    directory_info = DirectoryInfo(dlu_data_directory, calculate_checksums = False)
     file_list = None
     if directory_info.file_count == 0 and directory_info.subdir_count == 0:
         error_msg = "Error: package " + package_id + " has no files or top level subdirectory"
@@ -92,9 +92,9 @@ def recall_dlu_package(package_id):
     if directory_info.file_count == 0 and directory_info.subdir_count == 1:
         contents = "".join(directory_info.dir_contents)
         top_level_subdir = package_id + "/" + contents
-        file_list = dlu_file_handler.match_files(top_level_subdir)
+        file_list = dlu_file_handler.match_files(top_level_subdir,False)
     else:
-        file_list = dlu_file_handler.match_files(package_id)
+        file_list = dlu_file_handler.match_files(package_id,False)
 
     dlu_files = []
     for file in directory_info.file_details:
@@ -117,4 +117,4 @@ def get_package_status(package_id):
     dlu_package_inventory = DLUPackageInventory()
     dlu_package_inventory.reconnect()
     status = dlu_package_inventory.get_package_status(package_id)
-    return status[0]["globus_dlu_status"] if len(status) > 0 and status[0]["globus_dlu_status"] is not None else ""
+    return status[0]["globus_dlu_status"] if len(status) > 0 and status[0]["globus_dlu_status"] is not None else ""

data_management/generate_sc_rnaseq_yaml.py

@@ -0,0 +1,38 @@
+import os
+import yaml
+import sys
+
+yamlData = {
+    "package_type": "Single-cell RNA-Seq",
+    "tis": "Michigan/Broad/Princeton",
+    "data_generators": "Rajasree Menon",
+    "dataset_description": ""
+}
+experiments = []
+
+if len(sys.argv) == 1:
+    print("Error. Please specify directory: python3 generate_sc_rnaseq_yaml.py /path/to/bulk/upload")
+    exit(1)
+
+dir = sys.argv[1]
+for root, dirs, files in os.walk(dir):
+    if root == dir:
+        continue
+    sample_id = os.path.split(root)[1]
+    experiment = {
+        "internal_experiment_id": sample_id,
+        "files": []
+    }
+    for file in files:
+        experiment['files'].append({
+            'redcap_id': sample_id,
+            'spectrack_sample_id': sample_id,
+            'relative_file_path_and_name': sample_id + '/' + file,
+            'file_metadata': ""
+        })
+    experiments.append({
+        "experiment": experiment
+    })
+yamlData["experiments"] = experiments
+with open(os.path.join(dir, 'bulk-manifest.yaml'), 'w') as file:
+    yaml.dump(yamlData, file)

data_management/lib/mysql_connection.py

@@ -86,7 +86,7 @@ def get_db_connection(self):
             self.database.get_warnings = True
             return self.database
         except Exception as error:
-            logger.error("Can't connect to MySQL: ", exec_info=error)
+            logger.exception("Can't connect to MySQL: ", error)
             os.sys.exit()
 
     def get_tableau_db_connection(self):
@@ -102,7 +102,7 @@ def get_tableau_db_connection(self):
             self.database.get_warnings = True
             return self.database
         except Exception as error:
-            logger.error("Can't connect to MySQL: ", exc_info=error)
+            logger.exception("Can't connect to MySQL: ", error)
             os.sys.exit()
 
     def insert_data(self, sql, data):
@@ -123,6 +123,20 @@ def insert_data(self, sql, data):
         finally:
             self.database.commit()
             self.cursor.close()
+
+    def insert_data_no_alert(self, sql, data):
+        try:
+            self.get_db_cursor()
+            self.cursor.execute(sql, data)
+            warning = self.cursor.fetchwarnings()
+            if warning is not None:
+                print(warning)
+        except:
+            message = f"Error: Cannot insert with query: {sql}; and the data: {data}"
+            logger.error(message)
+        finally:
+            self.database.commit()
+            self.cursor.close()
 
     def get_data(self, sql, query_data=None):
         try:
@@ -132,13 +146,12 @@ def get_data(self, sql, query_data=None):
             for row in self.cursor:
                 data.append(row)
             return data
-        except:
-            message = "Error: Can't get data_management data."
-            logger.error(message)
+        except Exception as error:
+            logger.error(str(error))
             requests.post(
                 slack_url,
                 headers={'Content-type': 'application/json', },
-                data='{"text":"' + message + '"}'
+                data='{"text":"' + "Error: " + str(error) + '"}'
             )
         finally:
             self.cursor.close()

data_management/main.py

@@ -43,6 +43,9 @@ def upsert_new_spectrack_specimens(self):
     def load_biopsy_tracking(self):
         return self.tableau.load_biopsy_tracking()
 
+    def load_biopsy_tracking_long(self):
+        return self.tableau.load_biopsy_tracking_long()
+
     def load_data_manager_data(self):
         return self.tableau.load_data_manager_data()
 
@@ -88,6 +91,7 @@ def update_biomarker_tracking_redcap_ids(self):
         if args.action == "insert" or args.action == "update":
             records_modified = main.load_biopsy_tracking()
             records_modified = records_modified + main.load_data_manager_data()
+            records_modified = records_modified + main.load_biopsy_tracking_long()
 
     if "records_modified" in locals():
         logger.info(f"{records_modified} records modified")

data_management/model/dlu_package.py

@@ -19,6 +19,7 @@ def __init__(self):
         self.dlu_protocol = None
         self.dlu_data_generators = None
         self.dlu_files = []
+        self.dlu_upload_type = None
         self.submitter_name = None
         self.known_specimen = None
         self.redcap_id = None
@@ -61,6 +62,7 @@ def get_dmd_dpi_tuple(self):
             self.dlu_subject_id,
             self.dlu_error,
             self.dlu_lfu,
+            self.dlu_upload_type,
             self.globus_dlu_status
         )
 

data_management/process_bulk_uploads.py

@@ -27,8 +27,8 @@ class ProcessBulkUploads:
     def __init__(self, data_directory: str, globus_only: bool = False, globus_root: str = None, preserve_path: bool = False, bypass_dup_check: bool = False):
         try:
             self.dlu_management = DluManagement()
-        except:
-            logger.error("There was a problem loading the Data Management library.")
+        except Exception as e:
+            logger.exception("There was a problem loading the Data Management library.", e)
         try:
             self.submitter = os.environ["mongo_submitter_id"]
             self.submitter_name = os.environ["submitter_name"]
@@ -68,20 +68,42 @@ def process_files(self, manifest_files_arr: list) -> list:
             logger.info(file_full_path)
             size = os.path.getsize(file_full_path)
             file_info = self.dlu_file_handler.split_path(file_path, self.preserve_path)
-            if file["file_metadata"] and "md5_hash" in file["file_metadata"]:
+            if "file_metadata" in file and "md5_hash" in file["file_metadata"]:
                 checksum = file["file_metadata"]["md5_hash"]
                 del file["file_metadata"]["md5_hash"]
             else:
                 checksum = calculate_checksum(file_full_path)
-            if file["file_metadata"]:
+            if "file_metadata" in file:
                 metadata = file["file_metadata"]
             else:
                 metadata = {}
             dlu_file = DLUFile(file_info["file_name"], file_info["file_path"], checksum, size, metadata)
             dlu_files.append(dlu_file)
         return dlu_files
 
+    def process_globus_only_files(self, manifest_files_arr: list) -> list:
+        logger.info("globus only file processing")
+        files = []
+        for file in manifest_files_arr:
+            file_path = file["relative_file_path_and_name"]
+            file_full_path = os.path.join(self.data_directory, file_path)
+            file_info = self.dlu_file_handler.split_path(file_path, self.preserve_path)
+            if "file_metadata" in file and "md5_hash" in file["file_metadata"]:
+                checksum = file["file_metadata"]["md5_hash"]
+                del file["file_metadata"]["md5_hash"]
+            if "file_metadata" in file:
+                metadata = file["file_metadata"]
+            else:
+                metadata = {}
+
+            # Since this is going directly to globus, we don't need to calc checksum or filesize, and we need
+            # the path to the file on disk to actually copy it
+            dlu_file = DLUFile(file_info["file_name"], file_full_path, '', 0, metadata)
+            files.append(dlu_file)
+        return files
+
     def process_bulk_uploads(self):
+        logger.info("in process bulk uploads")
         for manifest_name in MANIFEST_FILE_NAMES:
             manifest_file_path = os.path.join(self.data_directory, manifest_name)
             if os.path.isfile(manifest_file_path):
@@ -93,13 +115,14 @@ def process_bulk_uploads(self):
             manifest_data = yaml.safe_load(stream)
             if manifest_data["package_type"] == "EM Images":
                 package_type = PackageType.ELECTRON_MICROSCOPY
-            elif manifest_data["package_type"] == "Segmentation Masks":
+            elif manifest_data["package_type"] == "Segmentation Masks & Pathomics Vectors":
                 package_type = PackageType.SEGMENTATION
             elif manifest_data["package_type"] == "Multimodal Images":
                 package_type = PackageType.MULTI_MODAL
             elif manifest_data["package_type"] == "Single-cell RNA-Seq":
                 package_type = PackageType.SINGLE_CELL
             else:
+                logger.info("package type is: ", manifest_data["package_type"])
                 package_type = PackageType.OTHER
             if "tis" in manifest_data:
                 tis = manifest_data["tis"]
@@ -111,20 +134,24 @@ def process_bulk_uploads(self):
                 redcap_id = experiment["files"][0]["redcap_id"]
                 sample_id = experiment["files"][0]["spectrack_sample_id"]
                 if redcap_id and redcap_id.startswith("S-"):
+                    logger.info("found redcap id starting with S-")
                     sample_id = redcap_id
                     redcap_results = self.dlu_management.get_redcapid_by_subjectid(sample_id)
-                    if redcap_results is  not None and len(redcap_results) == 1:
+                    if redcap_results is not None and len(redcap_results) > 1:
                         redcap_id = redcap_results
                     else:
-                        redcap_id = ""
+                        redcap_id = None
 
                 if not sample_id:
                     sample_id = redcap_id
 
                 if (sample_id and len(self.dlu_management.get_participant_by_redcap_id(redcap_id)) > 0) or \
                         (self.globus_only and sample_id):
                     logger.info(f"Trying to add package for {redcap_id} / {sample_id}")
-                    dlu_file_list = self.process_files(experiment["files"])
+                    if self.globus_only:
+                        dlu_file_list = self.process_globus_only_files(experiment["files"])
+                    else:
+                        dlu_file_list = self.process_files(experiment["files"])
                     if package_type == PackageType.SEGMENTATION:
                         dlu_file_list.append(self.get_single_file(SEGMENTATION_README))
                         tis = "UFL"
@@ -150,6 +177,7 @@ def process_bulk_uploads(self):
                         package.dlu_version = 4
                         package.dlu_dataset_information_version = 1
                         package.dlu_error = 0
+                        package.dlu_upload_type = 'KPMP Biopsy'
                         if self.globus_only:
                             package.globus_dlu_status = None
                         else:

data_management/rebuild.sh

@@ -1,2 +1,2 @@
 python3 setup.py install --user
-docker build -t kingstonduo/data-management:1.8.1 .
+docker build -t kingstonduo/data-management:1.10 .