From 0ef3d981555ee6db00e52b0fe6f6197babe15c34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20Pl=C3=A9siat?= <plesiat@dkrz.de>
Date: Thu, 4 May 2023 17:52:12 +0200
Subject: [PATCH 01/49] Process all files in zip archive

---
 duck/clintai.py               |  6 ++--
 duck/processes/wps_clintai.py | 65 +++++++++++++++++++++++------------
 2 files changed, 46 insertions(+), 25 deletions(-)

diff --git a/duck/clintai.py b/duck/clintai.py
index e23545b..338c5e3 100644
--- a/duck/clintai.py
+++ b/duck/clintai.py
@@ -40,10 +40,10 @@ def write_clintai_cfg(base_dir, model_dir, eval_name, data_name, data_type, eval
 
 def run(dataset, dataset_name, variable_name, outdir, update_status):
     # data_type = info_models[dataset_name]["variable-name"]
-    (outdir / "masks").mkdir()
-    (outdir / "outputs").mkdir()
+    (outdir / "masks").mkdir(exist_ok=True)
+    (outdir / "outputs").mkdir(exist_ok=True)
     input_dir = outdir / "test"
-    input_dir.mkdir()
+    input_dir.mkdir(exist_ok=True)
     shutil.move(dataset.as_posix(), input_dir.as_posix())
     # print(f"dataset={dataset}")
     cfg_file = write_clintai_cfg(
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 3d43813..f8885f9 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -1,5 +1,6 @@
 from pathlib import Path
 from zipfile import ZipFile
+import os
 
 from pywps import Process
 from pywps import LiteralInput, ComplexInput, ComplexOutput
@@ -78,41 +79,61 @@ def _handler(self, request, response):
         dataset_name = request.inputs['dataset_name'][0].data
         file = request.inputs['file'][0].file
         variable_name = request.inputs['variable_name'][0].data
-        print()
 
         response.update_status('Prepare dataset ...', 0)
         workdir = Path(self.workdir)
 
+        zipfile = False
         if Path(file).suffix == ".zip":
             with ZipFile(file, 'r') as zip:
                 print("extraction zip file", workdir)
                 zip.extractall(workdir.as_posix())
+                zipfile = True
 
-        # only one dataset file
         try:
-            dataset_0 = list(workdir.rglob('*.nc'))[0]
+            datasets = sorted(workdir.rglob('*.nc'), key=os.path.getmtime)
         except Exception:
             raise ProcessError("Could not extract netcdf file.")
 
-        ds = xr.open_dataset(dataset_0)
-
-        vars = list(ds.keys())
-        if variable_name not in vars:
-            raise ProcessError("Could not find variable {} in dataset.".format(variable_name))
-
-        # response.update_status('Infilling ...', 20)
-        try:
-            clintai.run(
-                dataset_0,
-                dataset_name=dataset_name,
-                variable_name=variable_name,
-                outdir=workdir,
-                update_status=response.update_status)
-        except Exception as e:
-            raise ProcessError(str(e))
-
-        response.outputs["output"].file = workdir / "outputs" / str(dataset_0.stem+"_infilled.nc")
-        response.outputs["plot"].file = workdir / "outputs" / str(dataset_0.stem+"_combined.1_0.png")
+        ndata = len(datasets)
+        if ndata == 0:
+            raise ProcessError("Could not find netcdf files.")
+
+        istep = 100 / ndata
+        i = 0
+        for dataset in datasets:
+
+            ds = xr.open_dataset(dataset)
+
+            vars = list(ds.keys())
+            if variable_name not in vars:
+                raise ProcessError("Could not find variable {} in {}.".format(variable_name, dataset))
+
+            try:
+                clintai.run(
+                    dataset,
+                    dataset_name=dataset_name,
+                    variable_name=variable_name,
+                    outdir=workdir,
+                    update_status=[response.update_status, i, istep])
+            except Exception as e:
+                raise ProcessError(str(e))
+
+            i += 1
+
+        if zipfile:
+            outfile = ".".join(file.split(".")[:-1]) + "_infilled.zip"
+            outfile = workdir / "outputs" / outfile
+            infiles = sorted((workdir / "outputs").rglob('*_infilled.nc'), key=os.path.getmtime)
+
+            with ZipFile(outfile, 'w') as zip:
+                for infile in infiles:
+                    zip.write(infile, arcname=infile.as_posix().split("/")[-1])
+        else:
+            outfile = workdir / "outputs" / str(datasets[0].stem+"_infilled.nc")
+
+        response.outputs["output"].file = outfile
+        response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
 
         response.update_status('done.', 100)
         return response

From 9ca0ac862b2d4f660325f5de63caa078359b1674 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Tue, 9 May 2023 13:32:46 +0200
Subject: [PATCH 02/49] added prov dependency

---
 environment.yml  | 4 ++++
 requirements.txt | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/environment.yml b/environment.yml
index 36cf0fb..1421bd1 100644
--- a/environment.yml
+++ b/environment.yml
@@ -29,3 +29,7 @@ dependencies:
 # plot
 - cartopy>=0.20.2
 - numba>=0.55.1
+# provenance
+- prov>=2.0.0
+- pydot
+- graphviz
diff --git a/requirements.txt b/requirements.txt
index 255f252..a1d483b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,6 @@ pywps>=4.5.1,<4.6
 # climatereconstructionAI>=0.1.0
 craimodels @ git+https://github.com/climateintelligence/craimodels@main#egg=craimodels
 climatereconstructionAI @ git+https://github.com/FREVA-CLINT/climatereconstructionAI@clint#egg=climatereconstructionAI
+# provenance
+prov>=2.0.0
+pydot

From 9c670ffd25ef7ff947622ade7e6bd902dc381312 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Tue, 9 May 2023 13:48:38 +0200
Subject: [PATCH 03/49] added provenance module

---
 duck/provenance.py | 193 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 193 insertions(+)
 create mode 100644 duck/provenance.py

diff --git a/duck/provenance.py b/duck/provenance.py
new file mode 100644
index 0000000..5846abb
--- /dev/null
+++ b/duck/provenance.py
@@ -0,0 +1,193 @@
+import os
+import uuid
+import json
+from datetime import datetime
+import pathlib
+
+from prov.identifier import Namespace
+import prov.model as prov
+from prov.dot import prov_to_dot
+
+# prov namespace
+PROV_ORGANISATION = prov.PROV["Organization"]
+PROV_SOFTWARE_AGENT = prov.PROV["SoftwareAgent"]
+
+# provone namespace
+PROVONE = Namespace(
+    "provone", uri="http://purl.dataone.org/provone/2015/01/15/ontology#"
+)
+PROVONE_WORKFLOW = PROVONE["Workflow"]
+PROVONE_DATA = PROVONE["Data"]
+PROVONE_EXECUTION = PROVONE["Execution"]
+
+# dcterms namespace
+DCTERMS = Namespace("dcterms", uri="http://purl.org/dc/terms/")
+DCTERMS_SOURCE = DCTERMS["source"]
+
+# clint namespace
+CLINT = Namespace("clint", uri="urn:clint:")
+
+
+class Provenance(object):
+    def __init__(self, output_dir):
+        if isinstance(output_dir, pathlib.Path):
+            self.output_dir = output_dir
+        else:
+            self.output_dir = pathlib.Path(output_dir)
+        self.doc = None
+        self._identifier = None
+        self._workflow = None
+
+    @property
+    def identifier(self):
+        return self._identifier
+
+    def start(self, workflow=False):
+        # from climatereconstructionai import __version__ as crai_version
+        crai_version = "1.0.2"
+        from duck import __version__ as duck_version
+
+        self.doc = prov.ProvDocument()
+        self._identifier = uuid.uuid4()
+        self._workflow = None
+        # Declaring namespaces for various prefixes
+        self.doc.add_namespace(CLINT)
+        self.doc.add_namespace(PROVONE)
+        self.doc.add_namespace(DCTERMS)
+        # Define entities
+        project_clint = self.doc.agent(
+            CLINT["CLINT"],
+            {
+                prov.PROV_TYPE: PROV_ORGANISATION,
+                prov.PROV_LABEL: "CLINT - climate intelligence",
+                DCTERMS_SOURCE: "https://climateintelligence.eu/",
+            },
+        )
+        self.sw_duck = self.doc.agent(
+            CLINT[f"duck_v{duck_version}"],
+            {
+                prov.PROV_TYPE: PROV_SOFTWARE_AGENT,
+                prov.PROV_LABEL: "Duck",
+                DCTERMS_SOURCE: f"https://github.com/climateintelligence/duck/releases/tag/v{duck_version}",
+            },
+        )
+        self.doc.wasAttributedTo(self.sw_duck, project_clint)
+        self.sw_crai = self.doc.agent(
+            CLINT[f"crai_v{crai_version}"],
+            {
+                prov.PROV_TYPE: PROV_SOFTWARE_AGENT,
+                prov.PROV_LABEL: "CRAI",
+                DCTERMS_SOURCE: f"https://github.com/FREVA-CLINT/climatereconstructionAI/releases/tag/v{crai_version}",
+            },
+        )
+        # workflow
+        if workflow is True:
+            self._workflow = self.doc.entity(
+                CLINT[f"workflow_{self.identifier}"], {prov.PROV_TYPE: PROVONE_WORKFLOW}
+            )
+            orchestrate = self._execution_activity(
+                identifier=CLINT[f"orchestrate_{self.identifier}"],
+                label="orchestrate",
+                attributes={
+                    prov.PROV_ATTR_STARTTIME: datetime.now().isoformat(
+                        timespec="seconds"
+                    )
+                },
+            )
+            self.doc.wasAssociatedWith(
+                orchestrate, agent=self.sw_duck, plan=self._workflow
+            )
+
+    def stop(self):
+        if self._workflow:
+            self._execution_activity(
+                identifier=CLINT[f"orchestrate_{self.identifier}"],
+                attributes={
+                    prov.PROV_ATTR_ENDTIME: datetime.now().isoformat(timespec="seconds")
+                },
+            )
+
+    def add_operator(self, operator, parameters, collection, output):
+        attributes = {}
+        for param in [
+            "time",
+            "time_components",
+            "area",
+            "level",
+            "dims",
+            "freq",
+            "apply_fixes",
+        ]:
+            if param in parameters:
+                value = parameters[param]
+                if isinstance(value, list):
+                    value = ",".join(value)
+                attributes[CLINT[param]] = value
+        op = self._execution_activity(
+            identifier=CLINT[f"{operator}_{uuid.uuid4()}"],
+            label=operator,
+            attributes=attributes,
+        )
+        # input data
+        ds_in = os.path.basename(collection[0])
+        op_input = self._data_entitiy(identifier=CLINT[ds_in], label=ds_in)
+        # operator started by crai
+        if self._workflow:
+            self.doc.wasAssociatedWith(op, agent=self.sw_crai, plan=self._workflow)
+        else:
+            self.doc.start(op, starter=self.sw_crai, trigger=self.sw_duck)
+        # Generated output file
+        for out in output:
+            ds_out = os.path.basename(out)
+            op_output = self._data_entitiy(identifier=CLINT[ds_out], label=ds_out)
+            self.doc.wasDerivedFrom(op_output, op_input, activity=op)
+
+    def _data_entitiy(self, identifier, label=None):
+        records = self.doc.get_record(identifier)
+        if records:
+            entity = records[0]
+        else:
+            entity = self.doc.entity(identifier)
+            entity.add_attributes(
+                {
+                    prov.PROV_TYPE: PROVONE_DATA,
+                    prov.PROV_LABEL: label or "data",
+                }
+            )
+        return entity
+
+    def _execution_activity(self, identifier, label=None, attributes=None):
+        records = self.doc.get_record(identifier)
+        if records:
+            activity = records[0]
+        else:
+            activity = self.doc.activity(identifier)
+            activity.add_attributes(
+                {
+                    prov.PROV_TYPE: PROVONE_EXECUTION,
+                    prov.PROV_LABEL: label or "operator",
+                }
+            )
+        if attributes:
+            activity.add_attributes(attributes)
+        return activity
+
+    def write_json(self):
+        outfile = self.output_dir / "provenance.json"
+        self.doc.serialize(outfile.as_posix(), format="json")
+        return outfile
+
+    def write_png(self):
+        outfile = self.output_dir / "provenance.png"
+        figure = prov_to_dot(self.doc)
+        figure.write_png(outfile.as_posix())
+        return outfile
+
+    def get_provn(self):
+        return self.doc.get_provn()
+
+    def dump_json(self):
+        return self.doc.serialize(indent=2)
+
+    def json(self):
+        return json.loads(self.dump_json())

From f6707a09f5ec6893132f6f27bbdb67e9f2d41f72 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Tue, 9 May 2023 14:57:59 +0200
Subject: [PATCH 04/49] added prov

---
 duck/processes/wps_clintai.py | 47 +++++++++++++++++++++++++++++------
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index f8885f9..d2dd188 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -10,6 +10,7 @@
 
 import craimodels
 from duck import clintai
+from duck.provenance import Provenance
 import xarray as xr
 
 import logging
@@ -48,6 +49,20 @@ def __init__(self):
                           # abstract='Plot of original input file. First timestep.',
                           as_reference=True,
                           supported_formats=[FORMAT_PNG]),
+            ComplexOutput(
+                "prov",
+                "Provenance",
+                abstract="Provenance document using W3C standard.",
+                as_reference=True,
+                supported_formats=[FORMATS.JSON],
+            ),
+            ComplexOutput(
+                "prov_plot",
+                "Provenance Diagram",
+                abstract="Provenance document as diagram.",
+                as_reference=True,
+                supported_formats=[FORMAT_PNG],
+            ),
         ]
 
         super(ClintAI, self).__init__(
@@ -82,6 +97,9 @@ def _handler(self, request, response):
 
         response.update_status('Prepare dataset ...', 0)
         workdir = Path(self.workdir)
+        # prov
+        prov = Provenance(self.workdir)
+        prov.start(workflow=True)
 
         zipfile = False
         if Path(file).suffix == ".zip":
@@ -110,12 +128,13 @@ def _handler(self, request, response):
                 raise ProcessError("Could not find variable {} in {}.".format(variable_name, dataset))
 
             try:
-                clintai.run(
-                    dataset,
-                    dataset_name=dataset_name,
-                    variable_name=variable_name,
-                    outdir=workdir,
-                    update_status=[response.update_status, i, istep])
+                # clintai.run(
+                #     dataset,
+                #     dataset_name=dataset_name,
+                #     variable_name=variable_name,
+                #     outdir=workdir,
+                #     update_status=[response.update_status, i, istep])
+                pass
             except Exception as e:
                 raise ProcessError(str(e))
 
@@ -132,8 +151,20 @@ def _handler(self, request, response):
         else:
             outfile = workdir / "outputs" / str(datasets[0].stem+"_infilled.nc")
 
-        response.outputs["output"].file = outfile
-        response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
+        # response.outputs["output"].file = outfile
+        response.outputs["output"].data = "test"
+        # response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
+        response.outputs["plot"].data = "test"
+
+        # prov
+        prov.add_operator(
+            "crai", {}, 
+            [datasets[0].as_posix()], 
+            [f"{datasets[0].as_posix()}_infilled.nc"]
+        )
+        prov.stop()
+        response.outputs["prov"].file = prov.write_json()
+        response.outputs["prov_plot"].file = prov.write_png()
 
         response.update_status('done.', 100)
         return response

From 4314b62a18285383f539fd6a21b4c383f0b017a2 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Tue, 9 May 2023 15:54:24 +0200
Subject: [PATCH 05/49] added duck stats

---
 duck/data_stats.py            | 44 +++++++++++++++++++++++++++++++++++
 duck/processes/wps_clintai.py |  8 ++++++-
 duck/provenance.py            |  5 ++--
 3 files changed, 54 insertions(+), 3 deletions(-)
 create mode 100644 duck/data_stats.py

diff --git a/duck/data_stats.py b/duck/data_stats.py
new file mode 100644
index 0000000..01a1c11
--- /dev/null
+++ b/duck/data_stats.py
@@ -0,0 +1,44 @@
+import xarray as xr
+import matplotlib.pyplot as plt
+import numpy as np
+
+def get_stats(data):
+    return {"min": np.nanmin(data), "max": np.nanmax(data), "mean": np.nanmean(data), "std": np.nanstd(data)}
+
+def data_stats(filename, var, nbins=100):
+
+    ds = xr.open_dataset(filename)
+
+    vstats = get_stats(ds[var].values)
+    bins = np.linspace(vstats["min"], vstats["max"], num=nbins + 1)
+
+    ntime, nlon, nlat = ds[var].shape
+    mratio = np.zeros(ntime)
+    hist = np.zeros((ntime, nbins))
+    for i in range(ntime):
+        a = ds[var].values[i]
+        idx = ~np.isnan(a)
+        mratio[i] = idx.sum()
+        a = np.histogram(a[idx], bins=bins)[0]
+        hist[i] = a / max(a)
+
+    mratio = 1 - mratio / (nlon * nlat)
+
+    # It would be great to store the distribution graph in a database
+    plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
+    ax = plt.gca()
+    ax.grid(color='gray', linestyle='-.', linewidth=1)
+    plt.xlabel(var)
+    plt.ylabel("Timesteps")
+    plt.savefig("histime.png", dpi=50)
+
+    # The following information should be stored in a database
+    print("Attrs:", ds.attrs)
+    print("Dims:",ds.dims)
+    print("Vars:", ds.variables)
+    print("Vstats:", vstats)
+    print("Mstats:", get_stats(mratio))
+
+filename = "data_tas_mon-gl-72_hadcrut4_observation_all_1850-2018.nc"
+var = "tas"
+data_stats(filename, var)
\ No newline at end of file
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index d2dd188..e8305ef 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -157,8 +157,14 @@ def _handler(self, request, response):
         response.outputs["plot"].data = "test"
 
         # prov
+        ds = xr.open_dataset(datasets[0])
         prov.add_operator(
-            "crai", {}, 
+            "crai", 
+            {
+                "dataset_name": dataset_name,
+                "variable_name": variable_name,
+                "dims": str(dict(ds.dims))
+            }, 
             [datasets[0].as_posix()], 
             [f"{datasets[0].as_posix()}_infilled.nc"]
         )
diff --git a/duck/provenance.py b/duck/provenance.py
index 5846abb..8ec21da 100644
--- a/duck/provenance.py
+++ b/duck/provenance.py
@@ -111,12 +111,13 @@ def add_operator(self, operator, parameters, collection, output):
         attributes = {}
         for param in [
             "time",
-            "time_components",
             "area",
             "level",
             "dims",
             "freq",
-            "apply_fixes",
+            "dataset_name",
+            "variable_name",
+            "dims"
         ]:
             if param in parameters:
                 value = parameters[param]

From 4d4bb84b4f4cf54109525970cda417e59f35c6bd Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 10:56:25 +0200
Subject: [PATCH 06/49] update prov

---
 duck/processes/wps_clintai.py | 3 +--
 duck/provenance.py            | 1 -
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index e8305ef..8f26d16 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -157,13 +157,12 @@ def _handler(self, request, response):
         response.outputs["plot"].data = "test"
 
         # prov
-        ds = xr.open_dataset(datasets[0])
+        # ds = xr.open_dataset(datasets[0])
         prov.add_operator(
             "crai", 
             {
                 "dataset_name": dataset_name,
                 "variable_name": variable_name,
-                "dims": str(dict(ds.dims))
             }, 
             [datasets[0].as_posix()], 
             [f"{datasets[0].as_posix()}_infilled.nc"]
diff --git a/duck/provenance.py b/duck/provenance.py
index 8ec21da..fdafbd4 100644
--- a/duck/provenance.py
+++ b/duck/provenance.py
@@ -117,7 +117,6 @@ def add_operator(self, operator, parameters, collection, output):
             "freq",
             "dataset_name",
             "variable_name",
-            "dims"
         ]:
             if param in parameters:
                 value = parameters[param]

From 69786993121ba26df3b99ce3f57362b606cd19f3 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 11:56:14 +0200
Subject: [PATCH 07/49] update stats

---
 duck/data_stats.py            | 39 +++++++++++++++++++----------------
 duck/processes/wps_clintai.py | 24 +++++++++++++++++++--
 environment.yml               |  2 +-
 3 files changed, 44 insertions(+), 21 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index 01a1c11..d5e513d 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -1,12 +1,13 @@
 import xarray as xr
 import matplotlib.pyplot as plt
 import numpy as np
+import yaml
+
 
 def get_stats(data):
     return {"min": np.nanmin(data), "max": np.nanmax(data), "mean": np.nanmean(data), "std": np.nanstd(data)}
 
-def data_stats(filename, var, nbins=100):
-
+def gen_data_stats(filename, var, nbins=100):
     ds = xr.open_dataset(filename)
 
     vstats = get_stats(ds[var].values)
@@ -24,21 +25,23 @@ def data_stats(filename, var, nbins=100):
 
     mratio = 1 - mratio / (nlon * nlat)
 
-    # It would be great to store the distribution graph in a database
-    plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
-    ax = plt.gca()
-    ax.grid(color='gray', linestyle='-.', linewidth=1)
-    plt.xlabel(var)
-    plt.ylabel("Timesteps")
-    plt.savefig("histime.png", dpi=50)
+    # TODO: It would be great to store the distribution graph in a database
+    if False:
+        plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
+        ax = plt.gca()
+        ax.grid(color='gray', linestyle='-.', linewidth=1)
+        plt.xlabel(var)
+        plt.ylabel("Timesteps")
+        plt.savefig("histime.png", dpi=50)
 
     # The following information should be stored in a database
-    print("Attrs:", ds.attrs)
-    print("Dims:",ds.dims)
-    print("Vars:", ds.variables)
-    print("Vstats:", vstats)
-    print("Mstats:", get_stats(mratio))
-
-filename = "data_tas_mon-gl-72_hadcrut4_observation_all_1850-2018.nc"
-var = "tas"
-data_stats(filename, var)
\ No newline at end of file
+    info = {}
+    info["Attrs"] = str(ds.attrs)
+    info["Dims"] = str(ds.dims)
+    info["Vars"] = str(ds.variables)
+    info["Vstats"] = str(vstats)
+    info["Mstats"] = str(get_stats(mratio))
+    with open("info.txt", "w") as f:
+        yaml.safe_dump(info, f)
+
+    return {"hist": "histime.png", "info": "info.txt"}
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 8f26d16..bac59d6 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -1,6 +1,7 @@
 from pathlib import Path
 from zipfile import ZipFile
 import os
+import xarray as xr
 
 from pywps import Process
 from pywps import LiteralInput, ComplexInput, ComplexOutput
@@ -11,7 +12,7 @@
 import craimodels
 from duck import clintai
 from duck.provenance import Provenance
-import xarray as xr
+from duck.data_stats import gen_data_stats
 
 import logging
 LOGGER = logging.getLogger("PYWPS")
@@ -63,6 +64,20 @@ def __init__(self):
                 as_reference=True,
                 supported_formats=[FORMAT_PNG],
             ),
+            ComplexOutput(
+                "info",
+                "Data Statistics",
+                abstract="Data Statistics of input file.",
+                as_reference=True,
+                supported_formats=[FORMATS.JSON],
+            ),
+            ComplexOutput(
+                "hist",
+                "Histogram ",
+                abstract="Histogram of input file.",
+                as_reference=True,
+                supported_formats=[FORMAT_PNG],
+            ),
         ]
 
         super(ClintAI, self).__init__(
@@ -156,8 +171,13 @@ def _handler(self, request, response):
         # response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
         response.outputs["plot"].data = "test"
 
+        # stats
+        stats = gen_data_stats(datasets[0].as_posix(), variable_name)
+        response.outputs["info"].file = stats["info"]
+        # response.outputs["hist"].file = stats["hist"]
+        response.outputs["hist"].data = "test"
+
         # prov
-        # ds = xr.open_dataset(datasets[0])
         prov.add_operator(
             "crai", 
             {
diff --git a/environment.yml b/environment.yml
index 1421bd1..8407c3a 100644
--- a/environment.yml
+++ b/environment.yml
@@ -4,7 +4,7 @@ channels:
 - pytorch
 dependencies:
 - pip
-- python>=3.7
+- python>=3.10
 - pywps>=4.5.1,<4.6
 - jinja2
 - click

From 61668f7b4e47e8f289fdf8a953e79b79b7755bcf Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 13:40:27 +0200
Subject: [PATCH 08/49] update stats

---
 duck/data_stats.py            | 98 +++++++++++++++++++++--------------
 duck/processes/wps_clintai.py |  9 ++--
 2 files changed, 64 insertions(+), 43 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index d5e513d..7a1eedd 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -2,46 +2,66 @@
 import matplotlib.pyplot as plt
 import numpy as np
 import yaml
-
+import pathlib
 
 def get_stats(data):
     return {"min": np.nanmin(data), "max": np.nanmax(data), "mean": np.nanmean(data), "std": np.nanstd(data)}
 
-def gen_data_stats(filename, var, nbins=100):
-    ds = xr.open_dataset(filename)
-
-    vstats = get_stats(ds[var].values)
-    bins = np.linspace(vstats["min"], vstats["max"], num=nbins + 1)
-
-    ntime, nlon, nlat = ds[var].shape
-    mratio = np.zeros(ntime)
-    hist = np.zeros((ntime, nbins))
-    for i in range(ntime):
-        a = ds[var].values[i]
-        idx = ~np.isnan(a)
-        mratio[i] = idx.sum()
-        a = np.histogram(a[idx], bins=bins)[0]
-        hist[i] = a / max(a)
-
-    mratio = 1 - mratio / (nlon * nlat)
-
-    # TODO: It would be great to store the distribution graph in a database
-    if False:
-        plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
-        ax = plt.gca()
-        ax.grid(color='gray', linestyle='-.', linewidth=1)
-        plt.xlabel(var)
-        plt.ylabel("Timesteps")
-        plt.savefig("histime.png", dpi=50)
-
-    # The following information should be stored in a database
-    info = {}
-    info["Attrs"] = str(ds.attrs)
-    info["Dims"] = str(ds.dims)
-    info["Vars"] = str(ds.variables)
-    info["Vstats"] = str(vstats)
-    info["Mstats"] = str(get_stats(mratio))
-    with open("info.txt", "w") as f:
-        yaml.safe_dump(info, f)
-
-    return {"hist": "histime.png", "info": "info.txt"}
+
+class DataStats(object):
+    def __init__(self, output_dir):
+        if isinstance(output_dir, pathlib.Path):
+            self.output_dir = output_dir
+        else:
+            self.output_dir = pathlib.Path(output_dir)
+        self.info = None
+
+   
+    def gen_data_stats(self, filename, var, nbins=100, with_hist=True):
+        ds = xr.open_dataset(filename)
+
+        vstats = get_stats(ds[var].values)
+        bins = np.linspace(vstats["min"], vstats["max"], num=nbins + 1)
+
+        ntime, nlon, nlat = ds[var].shape
+        mratio = np.zeros(ntime)
+        hist = np.zeros((ntime, nbins))
+        for i in range(ntime):
+            a = ds[var].values[i]
+            idx = ~np.isnan(a)
+            mratio[i] = idx.sum()
+            a = np.histogram(a[idx], bins=bins)[0]
+            hist[i] = a / max(a)
+
+        mratio = 1 - mratio / (nlon * nlat)
+
+        # TODO: It would be great to store the distribution graph in a database
+        if with_hist:
+            plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
+            ax = plt.gca()
+            ax.grid(color='gray', linestyle='-.', linewidth=1)
+            plt.xlabel(var)
+            plt.ylabel("Timesteps")
+            outfile = self.output_dir / "histime.png"
+            plt.savefig(outfile.as_posix(), dpi=50)
+
+        # The following information should be stored in a database
+        self.info = {}
+        self.info["Attrs"] = str(ds.attrs)
+        self.info["Dims"] = str(ds.dims)
+        self.info["Vars"] = str(ds.variables)
+        self.info["Vstats"] = str(vstats)
+        self.info["Mstats"] = str(get_stats(mratio))
+    
+    def write_json(self):
+        outfile = self.output_dir / "info.json"
+        with open(outfile.as_posix(), "w") as f:
+            yaml.safe_dump(self.info, f)
+        return outfile
+    
+    def write_png(self):
+        outfile = self.output_dir / "histime.png"
+        return outfile
+
+
+
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index bac59d6..0d78e68 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -12,7 +12,7 @@
 import craimodels
 from duck import clintai
 from duck.provenance import Provenance
-from duck.data_stats import gen_data_stats
+from duck.data_stats import DataStats
 
 import logging
 LOGGER = logging.getLogger("PYWPS")
@@ -172,9 +172,10 @@ def _handler(self, request, response):
         response.outputs["plot"].data = "test"
 
         # stats
-        stats = gen_data_stats(datasets[0].as_posix(), variable_name)
-        response.outputs["info"].file = stats["info"]
-        # response.outputs["hist"].file = stats["hist"]
+        datastats = DataStats(self.workdir)
+        datastats.gen_data_stats(datasets[0].as_posix(), variable_name, with_hist=False)
+        response.outputs["info"].file = datastats.write_json()
+        # response.outputs["hist"].file = datastats.write_png()
         response.outputs["hist"].data = "test"
 
         # prov

From e55af265123eccd4d6dca2445d7851300a9270c3 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 13:54:42 +0200
Subject: [PATCH 09/49] update stats

---
 duck/data_stats.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index 7a1eedd..ef75282 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -5,7 +5,12 @@
 import pathlib
 
 def get_stats(data):
-    return {"min": np.nanmin(data), "max": np.nanmax(data), "mean": np.nanmean(data), "std": np.nanstd(data)}
+    return {
+        "min": float(np.nanmin(data)), 
+        "max": float(np.nanmax(data)), 
+        "mean": float(np.nanmean(data)), 
+        "std": float(np.nanstd(data))
+    }
 
 
 class DataStats(object):
@@ -47,11 +52,12 @@ def gen_data_stats(self, filename, var, nbins=100, with_hist=True):
 
         # The following information should be stored in a database
         self.info = {}
-        self.info["Attrs"] = str(ds.attrs)
-        self.info["Dims"] = str(ds.dims)
-        self.info["Vars"] = str(ds.variables)
-        self.info["Vstats"] = str(vstats)
-        self.info["Mstats"] = str(get_stats(mratio))
+        self.info["Attrs"] = dict(ds.attrs)
+        self.info["Dims"] = dict(ds.dims)
+        # self.info["Vars"] = ds.variables
+        print(vstats)
+        self.info["Vstats"] = vstats
+        self.info["Mstats"] = get_stats(mratio)
     
     def write_json(self):
         outfile = self.output_dir / "info.json"

From da91eb44fddea8edb0d7ee90b9f59611a6826249 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 15:49:22 +0200
Subject: [PATCH 10/49] update stats

---
 duck/data_stats.py            | 23 +++++++++++------------
 duck/processes/wps_clintai.py | 14 +++++++-------
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index ef75282..a767227 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -22,7 +22,7 @@ def __init__(self, output_dir):
         self.info = None
 
    
-    def gen_data_stats(self, filename, var, nbins=100, with_hist=True):
+    def gen_data_stats(self, filename, var, nbins=100):
         ds = xr.open_dataset(filename)
 
         vstats = get_stats(ds[var].values)
@@ -41,21 +41,20 @@ def gen_data_stats(self, filename, var, nbins=100, with_hist=True):
         mratio = 1 - mratio / (nlon * nlat)
 
         # TODO: It would be great to store the distribution graph in a database
-        if with_hist:
-            plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
-            ax = plt.gca()
-            ax.grid(color='gray', linestyle='-.', linewidth=1)
-            plt.xlabel(var)
-            plt.ylabel("Timesteps")
-            outfile = self.output_dir / "histime.png"
-            plt.savefig(outfile.as_posix(), dpi=50)
+        plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
+        ax = plt.gca()
+        ax.grid(color='gray', linestyle='-.', linewidth=1)
+        plt.xlabel(var)
+        plt.ylabel("Timesteps")
+        outfile = self.output_dir / "histime.png"
+        plt.savefig(outfile.as_posix(), dpi=50)
 
         # The following information should be stored in a database
         self.info = {}
-        self.info["Attrs"] = dict(ds.attrs)
+        self.info["Attrs"] = str(ds.attrs)
         self.info["Dims"] = dict(ds.dims)
-        # self.info["Vars"] = ds.variables
-        print(vstats)
+        self.info["Vars"] = list(dict(ds.variables).keys())
+        # print(vstats)
         self.info["Vstats"] = vstats
         self.info["Mstats"] = get_stats(mratio)
     
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 0d78e68..7df8d4e 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -131,7 +131,14 @@ def _handler(self, request, response):
         ndata = len(datasets)
         if ndata == 0:
             raise ProcessError("Could not find netcdf files.")
+        
+        # stats
+        datastats = DataStats(self.workdir)
+        datastats.gen_data_stats(datasets[0].as_posix(), variable_name)
+        response.outputs["info"].file = datastats.write_json()
+        response.outputs["hist"].file = datastats.write_png()
 
+        # run crai
         istep = 100 / ndata
         i = 0
         for dataset in datasets:
@@ -171,13 +178,6 @@ def _handler(self, request, response):
         # response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
         response.outputs["plot"].data = "test"
 
-        # stats
-        datastats = DataStats(self.workdir)
-        datastats.gen_data_stats(datasets[0].as_posix(), variable_name, with_hist=False)
-        response.outputs["info"].file = datastats.write_json()
-        # response.outputs["hist"].file = datastats.write_png()
-        response.outputs["hist"].data = "test"
-
         # prov
         prov.add_operator(
             "crai", 

From e4eaf0386a1f7fe8399492a029947e26dfb0f74a Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 17:41:42 +0200
Subject: [PATCH 11/49] enable crai

---
 duck/processes/wps_clintai.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 7df8d4e..4417a58 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -150,13 +150,12 @@ def _handler(self, request, response):
                 raise ProcessError("Could not find variable {} in {}.".format(variable_name, dataset))
 
             try:
-                # clintai.run(
-                #     dataset,
-                #     dataset_name=dataset_name,
-                #     variable_name=variable_name,
-                #     outdir=workdir,
-                #     update_status=[response.update_status, i, istep])
-                pass
+                clintai.run(
+                    dataset,
+                    dataset_name=dataset_name,
+                    variable_name=variable_name,
+                    outdir=workdir,
+                    update_status=[response.update_status, i, istep])
             except Exception as e:
                 raise ProcessError(str(e))
 
@@ -173,10 +172,10 @@ def _handler(self, request, response):
         else:
             outfile = workdir / "outputs" / str(datasets[0].stem+"_infilled.nc")
 
-        # response.outputs["output"].file = outfile
-        response.outputs["output"].data = "test"
-        # response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
-        response.outputs["plot"].data = "test"
+        response.outputs["output"].file = outfile
+        # response.outputs["output"].data = "test"
+        response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
+        # response.outputs["plot"].data = "test"
 
         # prov
         prov.add_operator(

From 1c0c75df6cefbab15e671af0065644228e55ca4b Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 17:51:17 +0200
Subject: [PATCH 12/49] update stats

---
 duck/data_stats.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index a767227..be88098 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -51,7 +51,7 @@ def gen_data_stats(self, filename, var, nbins=100):
 
         # The following information should be stored in a database
         self.info = {}
-        self.info["Attrs"] = str(ds.attrs)
+        self.info["Attrs"] = dict(ds.attrs)
         self.info["Dims"] = dict(ds.dims)
         self.info["Vars"] = list(dict(ds.variables).keys())
         # print(vstats)
@@ -59,9 +59,9 @@ def gen_data_stats(self, filename, var, nbins=100):
         self.info["Mstats"] = get_stats(mratio)
     
     def write_json(self):
-        outfile = self.output_dir / "info.json"
+        outfile = self.output_dir / "info.yaml"
         with open(outfile.as_posix(), "w") as f:
-            yaml.safe_dump(self.info, f)
+            yaml.dump(self.info, f)
         return outfile
     
     def write_png(self):

From 68bad563a7f107ce079c6fe25db2d7d579385e1b Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 17:52:58 +0200
Subject: [PATCH 13/49] update stats

---
 duck/data_stats.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index be88098..9158cbd 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -59,7 +59,7 @@ def gen_data_stats(self, filename, var, nbins=100):
         self.info["Mstats"] = get_stats(mratio)
     
     def write_json(self):
-        outfile = self.output_dir / "info.yaml"
+        outfile = self.output_dir / "info.txt"
         with open(outfile.as_posix(), "w") as f:
             yaml.dump(self.info, f)
         return outfile

From 6628ee2be5a89db32279a05b088f2295a8cf328f Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 17:57:19 +0200
Subject: [PATCH 14/49] update stats

---
 duck/data_stats.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index 9158cbd..ab3f1aa 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -50,8 +50,14 @@ def gen_data_stats(self, filename, var, nbins=100):
         plt.savefig(outfile.as_posix(), dpi=50)
 
         # The following information should be stored in a database
+        attrs = {}
+        orig_attrs = dict(ds.attrs)
+        for key in orig_attrs:
+            value = isinstance(orig_attrs[key], str):
+            attrs[key] = value
+
         self.info = {}
-        self.info["Attrs"] = dict(ds.attrs)
+        self.info["Attrs"] = attrs
         self.info["Dims"] = dict(ds.dims)
         self.info["Vars"] = list(dict(ds.variables).keys())
         # print(vstats)

From 10e1d418517719b74ad4dc52d79959ce73022663 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Thu, 11 May 2023 17:59:03 +0200
Subject: [PATCH 15/49] fix stats

---
 duck/data_stats.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index ab3f1aa..2aa7af1 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -53,8 +53,9 @@ def gen_data_stats(self, filename, var, nbins=100):
         attrs = {}
         orig_attrs = dict(ds.attrs)
         for key in orig_attrs:
-            value = isinstance(orig_attrs[key], str):
-            attrs[key] = value
+            value = orig_attrs[key]
+            if isinstance(value, str):
+                attrs[key] = value
 
         self.info = {}
         self.info["Attrs"] = attrs

From 5b631dcec6178539d32819f017f54d9b1fd9a953 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Mon, 4 Sep 2023 16:45:56 +0200
Subject: [PATCH 16/49] updated provenance

---
 duck/db.py                    | 39 ++++++++++++++++++
 duck/processes/wps_clintai.py | 51 +++++------------------
 duck/provenance.py            | 76 +++++++++++++++++------------------
 environment.yml               |  4 ++
 requirements.txt              |  4 ++
 5 files changed, 96 insertions(+), 78 deletions(-)
 create mode 100644 duck/db.py

diff --git a/duck/db.py b/duck/db.py
new file mode 100644
index 0000000..1e36747
--- /dev/null
+++ b/duck/db.py
@@ -0,0 +1,39 @@
+from rdflib import Graph, URIRef
+from rdflib.plugins.sparql import prepareQuery
+
+# Provide the path to the SQLite database in the local folder
+DB_URL = "sqlite:///provenance.db"
+
+class GraphDB(object):
+    def __init__(self):
+        # Create a graph with a specific backend store
+        self.graph = Graph(
+            store="SQLAlchemy", 
+            identifier=URIRef("http://example.org/graph"))
+        self.graph.open(DB_URL, create=True)
+
+    def add(self, data):
+        new_graph = Graph()
+        new_graph.parse(data=data, format="turtle")
+
+        # add rdf to existing graph
+        for triple in new_graph:
+            self.graph.add(triple)
+        # Commit changes to the store
+        self.graph.commit()
+
+    def query(self, query_str):
+        namespaces = {
+            "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+            "foaf": "http://xmlns.com/foaf/0.1/",
+            "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
+            "prov": "http://www.w3.org/ns/prov#",
+            "provone": "http://purl.dataone.org/provone/2015/01/15/ontology#",
+            "dcterms": "http://purl.org/dc/terms/",
+            "clint": "urn:clint:"
+        }
+        query = prepareQuery(query_str, initNs=namespaces)
+        results = self.graph.query(query)
+        return results
+
+    
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 4417a58..b187937 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -50,34 +50,6 @@ def __init__(self):
                           # abstract='Plot of original input file. First timestep.',
                           as_reference=True,
                           supported_formats=[FORMAT_PNG]),
-            ComplexOutput(
-                "prov",
-                "Provenance",
-                abstract="Provenance document using W3C standard.",
-                as_reference=True,
-                supported_formats=[FORMATS.JSON],
-            ),
-            ComplexOutput(
-                "prov_plot",
-                "Provenance Diagram",
-                abstract="Provenance document as diagram.",
-                as_reference=True,
-                supported_formats=[FORMAT_PNG],
-            ),
-            ComplexOutput(
-                "info",
-                "Data Statistics",
-                abstract="Data Statistics of input file.",
-                as_reference=True,
-                supported_formats=[FORMATS.JSON],
-            ),
-            ComplexOutput(
-                "hist",
-                "Histogram ",
-                abstract="Histogram of input file.",
-                as_reference=True,
-                supported_formats=[FORMAT_PNG],
-            ),
         ]
 
         super(ClintAI, self).__init__(
@@ -107,15 +79,15 @@ def __init__(self):
 
     def _handler(self, request, response):
         dataset_name = request.inputs['dataset_name'][0].data
+        print(f"{dataset_name}")
         file = request.inputs['file'][0].file
+        print(f"{file}")
         variable_name = request.inputs['variable_name'][0].data
+        print(f"{variable_name}")
 
         response.update_status('Prepare dataset ...', 0)
         workdir = Path(self.workdir)
-        # prov
-        prov = Provenance(self.workdir)
-        prov.start(workflow=True)
-
+       
         zipfile = False
         if Path(file).suffix == ".zip":
             with ZipFile(file, 'r') as zip:
@@ -134,9 +106,9 @@ def _handler(self, request, response):
         
         # stats
         datastats = DataStats(self.workdir)
-        datastats.gen_data_stats(datasets[0].as_posix(), variable_name)
-        response.outputs["info"].file = datastats.write_json()
-        response.outputs["hist"].file = datastats.write_png()
+        # datastats.gen_data_stats(datasets[0].as_posix(), variable_name)
+        # response.outputs["info"].file = datastats.write_json()
+        # response.outputs["hist"].file = datastats.write_png()
 
         # run crai
         istep = 100 / ndata
@@ -173,11 +145,10 @@ def _handler(self, request, response):
             outfile = workdir / "outputs" / str(datasets[0].stem+"_infilled.nc")
 
         response.outputs["output"].file = outfile
-        # response.outputs["output"].data = "test"
         response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
-        # response.outputs["plot"].data = "test"
 
         # prov
+        prov = Provenance(self.workdir)
         prov.add_operator(
             "crai", 
             {
@@ -187,9 +158,9 @@ def _handler(self, request, response):
             [datasets[0].as_posix()], 
             [f"{datasets[0].as_posix()}_infilled.nc"]
         )
-        prov.stop()
-        response.outputs["prov"].file = prov.write_json()
-        response.outputs["prov_plot"].file = prov.write_png()
+        prov.store_rdf()
+        # prov end
+
 
         response.update_status('done.', 100)
         return response
diff --git a/duck/provenance.py b/duck/provenance.py
index fdafbd4..ebea6b4 100644
--- a/duck/provenance.py
+++ b/duck/provenance.py
@@ -8,6 +8,8 @@
 import prov.model as prov
 from prov.dot import prov_to_dot
 
+from duck.db import GraphDB
+
 # prov namespace
 PROV_ORGANISATION = prov.PROV["Organization"]
 PROV_SOFTWARE_AGENT = prov.PROV["SoftwareAgent"]
@@ -16,7 +18,6 @@
 PROVONE = Namespace(
     "provone", uri="http://purl.dataone.org/provone/2015/01/15/ontology#"
 )
-PROVONE_WORKFLOW = PROVONE["Workflow"]
 PROVONE_DATA = PROVONE["Data"]
 PROVONE_EXECUTION = PROVONE["Execution"]
 
@@ -36,20 +37,20 @@ def __init__(self, output_dir):
             self.output_dir = pathlib.Path(output_dir)
         self.doc = None
         self._identifier = None
-        self._workflow = None
+        self.init()
 
     @property
     def identifier(self):
         return self._identifier
 
-    def start(self, workflow=False):
+    def init(self):
         # from climatereconstructionai import __version__ as crai_version
         crai_version = "1.0.2"
-        from duck import __version__ as duck_version
+        # from duck import __version__ as duck_version
+        duck_version = "0.1.0"
 
         self.doc = prov.ProvDocument()
         self._identifier = uuid.uuid4()
-        self._workflow = None
         # Declaring namespaces for various prefixes
         self.doc.add_namespace(CLINT)
         self.doc.add_namespace(PROVONE)
@@ -80,34 +81,8 @@ def start(self, workflow=False):
                 DCTERMS_SOURCE: f"https://github.com/FREVA-CLINT/climatereconstructionAI/releases/tag/v{crai_version}",
             },
         )
-        # workflow
-        if workflow is True:
-            self._workflow = self.doc.entity(
-                CLINT[f"workflow_{self.identifier}"], {prov.PROV_TYPE: PROVONE_WORKFLOW}
-            )
-            orchestrate = self._execution_activity(
-                identifier=CLINT[f"orchestrate_{self.identifier}"],
-                label="orchestrate",
-                attributes={
-                    prov.PROV_ATTR_STARTTIME: datetime.now().isoformat(
-                        timespec="seconds"
-                    )
-                },
-            )
-            self.doc.wasAssociatedWith(
-                orchestrate, agent=self.sw_duck, plan=self._workflow
-            )
 
-    def stop(self):
-        if self._workflow:
-            self._execution_activity(
-                identifier=CLINT[f"orchestrate_{self.identifier}"],
-                attributes={
-                    prov.PROV_ATTR_ENDTIME: datetime.now().isoformat(timespec="seconds")
-                },
-            )
-
-    def add_operator(self, operator, parameters, collection, output):
+    def add_operator(self, operator, parameters, collection, output, start, end):
         attributes = {}
         for param in [
             "time",
@@ -115,6 +90,10 @@ def add_operator(self, operator, parameters, collection, output):
             "level",
             "dims",
             "freq",
+            "min",
+            "max",
+            "stddev",
+            "mean",
             "dataset_name",
             "variable_name",
         ]:
@@ -127,22 +106,21 @@ def add_operator(self, operator, parameters, collection, output):
             identifier=CLINT[f"{operator}_{uuid.uuid4()}"],
             label=operator,
             attributes=attributes,
+            start=start,
+            end=end
         )
         # input data
         ds_in = os.path.basename(collection[0])
         op_input = self._data_entitiy(identifier=CLINT[ds_in], label=ds_in)
         # operator started by crai
-        if self._workflow:
-            self.doc.wasAssociatedWith(op, agent=self.sw_crai, plan=self._workflow)
-        else:
-            self.doc.start(op, starter=self.sw_crai, trigger=self.sw_duck)
+        self.doc.start(op, starter=self.sw_crai, trigger=self.sw_duck)
         # Generated output file
         for out in output:
             ds_out = os.path.basename(out)
             op_output = self._data_entitiy(identifier=CLINT[ds_out], label=ds_out)
             self.doc.wasDerivedFrom(op_output, op_input, activity=op)
 
-    def _data_entitiy(self, identifier, label=None):
+    def _data_entitiy(self, identifier, label=None, value=None):
         records = self.doc.get_record(identifier)
         if records:
             entity = records[0]
@@ -152,11 +130,12 @@ def _data_entitiy(self, identifier, label=None):
                 {
                     prov.PROV_TYPE: PROVONE_DATA,
                     prov.PROV_LABEL: label or "data",
+                    prov.PROV_VALUE: value or label or "",
                 }
             )
         return entity
 
-    def _execution_activity(self, identifier, label=None, attributes=None):
+    def _execution_activity(self, identifier, label=None, attributes=None, start=None, end=None):
         records = self.doc.get_record(identifier)
         if records:
             activity = records[0]
@@ -170,6 +149,12 @@ def _execution_activity(self, identifier, label=None, attributes=None):
             )
         if attributes:
             activity.add_attributes(attributes)
+        if start and end:
+            time_attribures={
+                prov.PROV_ATTR_STARTTIME: start,
+                prov.PROV_ATTR_ENDTIME: end
+            }
+            activity.add_attributes(time_attribures)
         return activity
 
     def write_json(self):
@@ -185,9 +170,24 @@ def write_png(self):
 
     def get_provn(self):
         return self.doc.get_provn()
+    
+    def get_rdf(self):
+        # Serialize the PROV document to RDF format
+        rdf_data = self.doc.serialize(format="rdf", rdf_format="turtle")
+        return rdf_data
+    
+    def write_rdf(self):
+        outfile = self.output_dir / "provenance.ttl"
+        with open(outfile.as_posix(), "w") as f:
+            f.write(self.get_rdf())
+
+    def store_rdf(self):
+        graph_db = GraphDB()
+        graph_db.add(self.get_rdf())
 
     def dump_json(self):
         return self.doc.serialize(indent=2)
 
     def json(self):
         return json.loads(self.dump_json())
+    
\ No newline at end of file
diff --git a/environment.yml b/environment.yml
index 8407c3a..3289096 100644
--- a/environment.yml
+++ b/environment.yml
@@ -33,3 +33,7 @@ dependencies:
 - prov>=2.0.0
 - pydot
 - graphviz
+- rdflib
+- rdflib-sqlalchemy
+- sqlalchemy<2
+- pandas
diff --git a/requirements.txt b/requirements.txt
index a1d483b..246da52 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,3 +9,7 @@ climatereconstructionAI @ git+https://github.com/FREVA-CLINT/climatereconstructi
 # provenance
 prov>=2.0.0
 pydot
+rdflib
+rdflib-sqlalchemy
+sqlalchemy<2
+pandas

From b5687b1264095b25b97f679dedaaab072c322a44 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Mon, 4 Sep 2023 16:49:17 +0200
Subject: [PATCH 17/49] update prov

---
 duck/processes/wps_clintai.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index b187937..f6cf8cd 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -1,6 +1,7 @@
 from pathlib import Path
 from zipfile import ZipFile
 import os
+from datetime import datetime
 import xarray as xr
 
 from pywps import Process
@@ -78,6 +79,8 @@ def __init__(self):
         )
 
     def _handler(self, request, response):
+        start_time = datetime.now().isoformat(timespec="seconds")
+
         dataset_name = request.inputs['dataset_name'][0].data
         print(f"{dataset_name}")
         file = request.inputs['file'][0].file
@@ -105,7 +108,7 @@ def _handler(self, request, response):
             raise ProcessError("Could not find netcdf files.")
         
         # stats
-        datastats = DataStats(self.workdir)
+        # datastats = DataStats(self.workdir)
         # datastats.gen_data_stats(datasets[0].as_posix(), variable_name)
         # response.outputs["info"].file = datastats.write_json()
         # response.outputs["hist"].file = datastats.write_png()
@@ -148,15 +151,21 @@ def _handler(self, request, response):
         response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
 
         # prov
+        end_time = datetime.now().isoformat(timespec="seconds")
         prov = Provenance(self.workdir)
         prov.add_operator(
             "crai", 
             {
                 "dataset_name": dataset_name,
                 "variable_name": variable_name,
+                "min": 10,
+                "max": 20,
+                "stddev": 2,
             }, 
             [datasets[0].as_posix()], 
-            [f"{datasets[0].as_posix()}_infilled.nc"]
+            [f"{datasets[0].as_posix()}_infilled.nc"],
+            start_time,
+            end_time,
         )
         prov.store_rdf()
         # prov end

From db90e853b249ff833cda5ba5daaec2b9905530ae Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Mon, 4 Sep 2023 17:10:26 +0200
Subject: [PATCH 18/49] using dummy code for testing

---
 .gitignore                    |  3 +++
 duck/processes/wps_clintai.py | 24 ++++++++++++++++++------
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index dcdcdb5..167a59d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -111,3 +111,6 @@ testdata.json
 
 # macos
 .DS_Store
+
+# provenance
+provenance.db
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index f6cf8cd..83d00fb 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -125,12 +125,24 @@ def _handler(self, request, response):
                 raise ProcessError("Could not find variable {} in {}.".format(variable_name, dataset))
 
             try:
-                clintai.run(
-                    dataset,
-                    dataset_name=dataset_name,
-                    variable_name=variable_name,
-                    outdir=workdir,
-                    update_status=[response.update_status, i, istep])
+                # dummy start
+                outputs_path =  workdir / "outputs" 
+                outputs_path.mkdir()
+                outfile = workdir / "outputs" / str(datasets[0].stem+"_infilled.nc")
+                print(outfile)
+                with outfile.open(mode='w') as file:
+                    file.write("dummy\n")
+                plotfile = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
+                print(plotfile)
+                with plotfile.open(mode='w') as file:
+                    file.write("dummy plot\n")
+                # dummy end
+                # clintai.run(
+                #     dataset,
+                #     dataset_name=dataset_name,
+                #     variable_name=variable_name,
+                #     outdir=workdir,
+                #     update_status=[response.update_status, i, istep])
             except Exception as e:
                 raise ProcessError(str(e))
 

From 70d28c8f6394cbeba464e548ac8c045defd37d14 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Mon, 4 Sep 2023 17:36:57 +0200
Subject: [PATCH 19/49] added dashboard process

---
 duck/db.py                      |  2 +-
 duck/processes/__init__.py      |  2 +
 duck/processes/wps_dashboard.py | 89 +++++++++++++++++++++++++++++++++
 duck/query.py                   | 56 +++++++++++++++++++++
 4 files changed, 148 insertions(+), 1 deletion(-)
 create mode 100644 duck/processes/wps_dashboard.py
 create mode 100644 duck/query.py

diff --git a/duck/db.py b/duck/db.py
index 1e36747..fb9accd 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -2,7 +2,7 @@
 from rdflib.plugins.sparql import prepareQuery
 
 # Provide the path to the SQLite database in the local folder
-DB_URL = "sqlite:///provenance.db"
+DB_URL = "sqlite:///provenance.sqlite"
 
 class GraphDB(object):
     def __init__(self):
diff --git a/duck/processes/__init__.py b/duck/processes/__init__.py
index 19300c7..9bccdb2 100644
--- a/duck/processes/__init__.py
+++ b/duck/processes/__init__.py
@@ -1,5 +1,7 @@
 from .wps_clintai import ClintAI
+from .wps_dashboard import Dashboard
 
 processes = [
     ClintAI(),
+    Dashboard(),
 ]
diff --git a/duck/processes/wps_dashboard.py b/duck/processes/wps_dashboard.py
new file mode 100644
index 0000000..4aef146
--- /dev/null
+++ b/duck/processes/wps_dashboard.py
@@ -0,0 +1,89 @@
+from pathlib import Path
+
+from pywps import Process, ComplexOutput, Format
+
+from duck import query
+
+class Dashboard(Process):
+    def __init__(self):
+        # inputs = [
+        #     ComplexInput('input_csv', 'CSV File', supported_formats=[Format('text/csv')]),
+        # ]
+        outputs = [
+            ComplexOutput(
+                'report', 
+                'Generated HTML Report', 
+                as_reference=True,
+                supported_formats=[Format('text/html')]
+            ),
+        ]
+
+        super(Dashboard, self).__init__(
+            self._handler,
+            identifier='dashboard',
+            title='Generate HTML Report',
+            version='1.0',
+            abstract='Generate an HTML report from a provenance database.',
+            # inputs=inputs,
+            outputs=outputs,
+        )
+
+    def _handler(self, request, response):
+        workdir = Path(self.workdir)
+        # input_csv = request.inputs['input_csv'][0].file
+
+        # Query the provenance database ... result is a Pandas DataFrame
+        df = query.query()
+
+        # Generate an HTML report from the DataFrame
+        html_report = self.write_html(df, workdir)
+
+        response.outputs['report'].file = html_report
+        response.outputs['report'].output_format = Format('text/html')
+
+        return response
+    
+    def write_html(self, df, workdir):
+        # Convert the DataFrame to an HTML table
+        html_table = df.to_html(index=False)
+
+        # Define the HTML template
+        html_template = f"""
+        <!DOCTYPE html>
+        <html>
+        <head>
+            <title>Provenance Report</title>
+            <style>
+                table {{
+                    border-collapse: collapse;
+                    width: 100%;
+                    border: 1px solid #ddd;
+                }}
+
+                th, td {{
+                    text-align: left;
+                    padding: 8px;
+                }}
+
+                th {{
+                    background-color: #f2f2f2;
+                }}
+
+                tr:nth-child(even) {{
+                    background-color: #f2f2f2;
+                }}
+            </style>
+        </head>
+        <body>
+            <h1>Provenance Report</h1>
+            {html_table}
+        </body>
+        </html>
+        """
+
+        # Write the HTML template to a file
+        outfile = workdir / "provenance_report.html"
+        with outfile.open(mode='w') as file:
+            file.write(html_template)
+
+        return outfile
diff --git a/duck/query.py b/duck/query.py
new file mode 100644
index 0000000..f5b578a
--- /dev/null
+++ b/duck/query.py
@@ -0,0 +1,56 @@
+from duck.db import GraphDB
+import pandas as pd
+
+
+def query():
+    query_str = """
+        SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?min ?max ?stddev
+        WHERE {
+            ?exec rdf:type provone:Execution ;
+                rdfs:label ?process ;
+                clint:dataset_name ?dataset ;
+                clint:variable_name ?variable ;
+                prov:startedAtTime ?startTime ;
+                prov:endedAtTime ?endTime ;
+                clint:min ?min ;
+                clint:max ?max ;
+                clint:stddev ?stddev .
+
+            ?input rdf:type prov:Entity .
+        
+            ?output rdf:type prov:Entity ;
+                prov:qualifiedDerivation [ prov:entity ?input; prov:hadActivity ?exec ] .
+    }
+    """
+    graph_db = GraphDB()
+    results = graph_db.query(query_str)
+
+    data = []
+    for row in results:
+        # print(row)
+        process = row.process.split("/")[-1]
+        dataset = row.dataset.value
+        variable = row.variable.value
+        start_time = row.startTime.value
+        end_time = row.endTime.value
+        input = row.input.split("/")[-1]
+        input = input.split("urn:clint:")[-1]
+        output = row.output.split("/")[-1]
+        output = output.split("urn:clint:")[-1]
+        min = row.min.value
+        max = row.max.value
+        stddev = row.stddev.value
+        data.append({
+            "Process": process, 
+            "Dataset": dataset,
+            "Variable": variable, 
+            "Start Time": start_time, 
+            "End Time": end_time,
+            "Input": input,
+            "Output": output,
+            "Min": min,
+            "Max": max,
+            "StdDev": stddev,
+        })
+    df = pd.DataFrame(data)
+    return df

From 98f5f2e0a09d2cbe50ec173c5fce47a9b8074401 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Mon, 4 Sep 2023 18:05:11 +0200
Subject: [PATCH 20/49] added stats to query

---
 duck/data_stats.py              | 15 ++++++-----
 duck/processes/wps_clintai.py   | 46 ++++++++++++++++++---------------
 duck/processes/wps_dashboard.py | 18 +++++++++----
 duck/query.py                   |  5 +++-
 4 files changed, 50 insertions(+), 34 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index 2aa7af1..8ffe5c0 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -41,13 +41,14 @@ def gen_data_stats(self, filename, var, nbins=100):
         mratio = 1 - mratio / (nlon * nlat)
 
         # TODO: It would be great to store the distribution graph in a database
-        plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
-        ax = plt.gca()
-        ax.grid(color='gray', linestyle='-.', linewidth=1)
-        plt.xlabel(var)
-        plt.ylabel("Timesteps")
-        outfile = self.output_dir / "histime.png"
-        plt.savefig(outfile.as_posix(), dpi=50)
+        if False:
+            plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
+            ax = plt.gca()
+            ax.grid(color='gray', linestyle='-.', linewidth=1)
+            plt.xlabel(var)
+            plt.ylabel("Timesteps")
+            outfile = self.output_dir / "histime.png"
+            plt.savefig(outfile.as_posix(), dpi=50)
 
         # The following information should be stored in a database
         attrs = {}
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 83d00fb..11b4bcd 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -25,6 +25,18 @@
 models_list = list(craimodels.info_models().keys())
 
 
+def dummy(workdir, datasets):
+    outputs_path =  workdir / "outputs" 
+    outputs_path.mkdir()
+    outfile = workdir / "outputs" / str(datasets[0].stem+"_infilled.nc")
+    print(outfile)
+    with outfile.open(mode='w') as file:
+        file.write("dummy\n")
+    plotfile = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
+    print(plotfile)
+    with plotfile.open(mode='w') as file:
+        file.write("dummy plot\n")
+
 class ClintAI(Process):
     def __init__(self):
         inputs = [
@@ -107,14 +119,8 @@ def _handler(self, request, response):
         if ndata == 0:
             raise ProcessError("Could not find netcdf files.")
         
-        # stats
-        # datastats = DataStats(self.workdir)
-        # datastats.gen_data_stats(datasets[0].as_posix(), variable_name)
-        # response.outputs["info"].file = datastats.write_json()
-        # response.outputs["hist"].file = datastats.write_png()
-
         # run crai
-        istep = 100 / ndata
+        # istep = 100 / ndata
         i = 0
         for dataset in datasets:
 
@@ -126,17 +132,9 @@ def _handler(self, request, response):
 
             try:
                 # dummy start
-                outputs_path =  workdir / "outputs" 
-                outputs_path.mkdir()
-                outfile = workdir / "outputs" / str(datasets[0].stem+"_infilled.nc")
-                print(outfile)
-                with outfile.open(mode='w') as file:
-                    file.write("dummy\n")
-                plotfile = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
-                print(plotfile)
-                with plotfile.open(mode='w') as file:
-                    file.write("dummy plot\n")
-                # dummy end
+                dummy(workdir, datasets)
+                # end
+
                 # clintai.run(
                 #     dataset,
                 #     dataset_name=dataset_name,
@@ -162,6 +160,11 @@ def _handler(self, request, response):
         response.outputs["output"].file = outfile
         response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
 
+        # stats
+        datastats = DataStats(self.workdir)
+        datastats.gen_data_stats(datasets[0].as_posix(), variable_name)
+        stats = datastats.info["Mstats"]
+
         # prov
         end_time = datetime.now().isoformat(timespec="seconds")
         prov = Provenance(self.workdir)
@@ -170,9 +173,10 @@ def _handler(self, request, response):
             {
                 "dataset_name": dataset_name,
                 "variable_name": variable_name,
-                "min": 10,
-                "max": 20,
-                "stddev": 2,
+                "min": stats["min"],
+                "max": stats["max"],
+                "mean": stats["mean"],
+                "stddev": stats["std"],
             }, 
             [datasets[0].as_posix()], 
             [f"{datasets[0].as_posix()}_infilled.nc"],
diff --git a/duck/processes/wps_dashboard.py b/duck/processes/wps_dashboard.py
index 4aef146..784fa1d 100644
--- a/duck/processes/wps_dashboard.py
+++ b/duck/processes/wps_dashboard.py
@@ -1,14 +1,22 @@
 from pathlib import Path
 
-from pywps import Process, ComplexOutput, Format
+from pywps import Process, LiteralInput, ComplexOutput, Format
 
 from duck import query
 
 class Dashboard(Process):
     def __init__(self):
-        # inputs = [
-        #     ComplexInput('input_csv', 'CSV File', supported_formats=[Format('text/csv')]),
-        # ]
+        inputs = [
+            LiteralInput(
+                "time",
+                "Time Period",
+                abstract="The time period for the report seperated by /"
+                "Example: 2023-09-01/2023-09-30",
+                data_type="string",
+                min_occurs=0,
+                max_occurs=1,
+            ),
+        ]
         outputs = [
             ComplexOutput(
                 'report', 
@@ -24,7 +32,7 @@ def __init__(self):
             title='Generate HTML Report',
             version='1.0',
             abstract='Generate an HTML report from a provenance database.',
-            # inputs=inputs,
+            inputs=inputs,
             outputs=outputs,
         )
 
diff --git a/duck/query.py b/duck/query.py
index f5b578a..8a59bc3 100644
--- a/duck/query.py
+++ b/duck/query.py
@@ -4,7 +4,7 @@
 
 def query():
     query_str = """
-        SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?min ?max ?stddev
+        SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?min ?max ?mean ?stddev
         WHERE {
             ?exec rdf:type provone:Execution ;
                 rdfs:label ?process ;
@@ -14,6 +14,7 @@ def query():
                 prov:endedAtTime ?endTime ;
                 clint:min ?min ;
                 clint:max ?max ;
+                clint:mean ?mean ;
                 clint:stddev ?stddev .
 
             ?input rdf:type prov:Entity .
@@ -39,6 +40,7 @@ def query():
         output = output.split("urn:clint:")[-1]
         min = row.min.value
         max = row.max.value
+        mean = row.mean.value
         stddev = row.stddev.value
         data.append({
             "Process": process, 
@@ -50,6 +52,7 @@ def query():
             "Output": output,
             "Min": min,
             "Max": max,
+            "Mean": mean,
             "StdDev": stddev,
         })
     df = pd.DataFrame(data)

From 4604c7f120dd324171fe374cf87b280a77e5b862 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Mon, 4 Sep 2023 19:02:56 +0200
Subject: [PATCH 21/49] added stats info to prov

---
 duck/data_stats.py            | 3 ++-
 duck/processes/wps_clintai.py | 2 ++
 duck/provenance.py            | 1 +
 duck/query.py                 | 7 +++++--
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index 8ffe5c0..a259c22 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -62,9 +62,10 @@ def gen_data_stats(self, filename, var, nbins=100):
         self.info["Attrs"] = attrs
         self.info["Dims"] = dict(ds.dims)
         self.info["Vars"] = list(dict(ds.variables).keys())
-        # print(vstats)
         self.info["Vstats"] = vstats
         self.info["Mstats"] = get_stats(mratio)
+        # self.info["Hist"] = hist
+        print(self.info)
     
     def write_json(self):
         outfile = self.output_dir / "info.txt"
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 11b4bcd..2c0caa1 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -2,6 +2,7 @@
 from zipfile import ZipFile
 import os
 from datetime import datetime
+import json
 import xarray as xr
 
 from pywps import Process
@@ -177,6 +178,7 @@ def _handler(self, request, response):
                 "max": stats["max"],
                 "mean": stats["mean"],
                 "stddev": stats["std"],
+                "info": json.dumps(datastats.info, separators=(',', ':')),
             }, 
             [datasets[0].as_posix()], 
             [f"{datasets[0].as_posix()}_infilled.nc"],
diff --git a/duck/provenance.py b/duck/provenance.py
index ebea6b4..5200fac 100644
--- a/duck/provenance.py
+++ b/duck/provenance.py
@@ -94,6 +94,7 @@ def add_operator(self, operator, parameters, collection, output, start, end):
             "max",
             "stddev",
             "mean",
+            "info",
             "dataset_name",
             "variable_name",
         ]:
diff --git a/duck/query.py b/duck/query.py
index 8a59bc3..f0f953d 100644
--- a/duck/query.py
+++ b/duck/query.py
@@ -4,7 +4,7 @@
 
 def query():
     query_str = """
-        SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?min ?max ?mean ?stddev
+        SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?min ?max ?mean ?stddev ?info
         WHERE {
             ?exec rdf:type provone:Execution ;
                 rdfs:label ?process ;
@@ -15,7 +15,8 @@ def query():
                 clint:min ?min ;
                 clint:max ?max ;
                 clint:mean ?mean ;
-                clint:stddev ?stddev .
+                clint:stddev ?stddev ;
+                clint:info ?info .
 
             ?input rdf:type prov:Entity .
         
@@ -42,6 +43,7 @@ def query():
         max = row.max.value
         mean = row.mean.value
         stddev = row.stddev.value
+        info = row.info.value
         data.append({
             "Process": process, 
             "Dataset": dataset,
@@ -54,6 +56,7 @@ def query():
             "Max": max,
             "Mean": mean,
             "StdDev": stddev,
+            "Info": info,
         })
     df = pd.DataFrame(data)
     return df

From cbca5c2074483dfde4c421e35337d12b1540c20f Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 10:51:15 +0200
Subject: [PATCH 22/49] fixed matplot

---
 duck/processes/wps_clintai.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 2c0caa1..8b2c03a 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -5,6 +5,9 @@
 import json
 import xarray as xr
 
+import matplotlib
+matplotlib.use('agg')
+
 from pywps import Process
 from pywps import LiteralInput, ComplexInput, ComplexOutput
 from pywps import FORMATS, Format
@@ -121,7 +124,7 @@ def _handler(self, request, response):
             raise ProcessError("Could not find netcdf files.")
         
         # run crai
-        # istep = 100 / ndata
+        istep = 100 / ndata
         i = 0
         for dataset in datasets:
 
@@ -133,15 +136,15 @@ def _handler(self, request, response):
 
             try:
                 # dummy start
-                dummy(workdir, datasets)
+                # dummy(workdir, datasets)
                 # end
 
-                # clintai.run(
-                #     dataset,
-                #     dataset_name=dataset_name,
-                #     variable_name=variable_name,
-                #     outdir=workdir,
-                #     update_status=[response.update_status, i, istep])
+                clintai.run(
+                    dataset,
+                    dataset_name=dataset_name,
+                    variable_name=variable_name,
+                    outdir=workdir,
+                    update_status=[response.update_status, i, istep])
             except Exception as e:
                 raise ProcessError(str(e))
 
@@ -163,7 +166,7 @@ def _handler(self, request, response):
 
         # stats
         datastats = DataStats(self.workdir)
-        datastats.gen_data_stats(datasets[0].as_posix(), variable_name)
+        datastats.gen_data_stats(outfile.as_posix(), variable_name)
         stats = datastats.info["Mstats"]
 
         # prov

From 3f9156593a595d76b2d992ef8e9cb95ef6c7808c Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 14:14:35 +0200
Subject: [PATCH 23/49] updated report with histogram

---
 duck/data_stats.py              | 37 ++++++++++++++++++++++-----------
 duck/processes/wps_clintai.py   |  1 +
 duck/processes/wps_dashboard.py |  2 +-
 duck/provenance.py              |  1 +
 duck/query.py                   | 37 ++++++++++++++++++++++++---------
 5 files changed, 55 insertions(+), 23 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index a259c22..fd24d8b 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -3,6 +3,11 @@
 import numpy as np
 import yaml
 import pathlib
+import io
+import base64
+
+# import matplotlib
+# matplotlib.use('agg')
 
 def get_stats(data):
     return {
@@ -20,6 +25,7 @@ def __init__(self, output_dir):
         else:
             self.output_dir = pathlib.Path(output_dir)
         self.info = None
+        self.histogram = None
 
    
     def gen_data_stats(self, filename, var, nbins=100):
@@ -41,14 +47,28 @@ def gen_data_stats(self, filename, var, nbins=100):
         mratio = 1 - mratio / (nlon * nlat)
 
         # TODO: It would be great to store the distribution graph in a database
-        if False:
+        if True:
+            plt.close()
             plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
             ax = plt.gca()
             ax.grid(color='gray', linestyle='-.', linewidth=1)
             plt.xlabel(var)
             plt.ylabel("Timesteps")
-            outfile = self.output_dir / "histime.png"
-            plt.savefig(outfile.as_posix(), dpi=50)
+            # outfile = self.output_dir / "histogram.png"
+            # print(f"histogram: {outfile}")
+            # plt.savefig(outfile.as_posix(), dpi=50)
+            # store as base64
+            # Save the plot to a BytesIO object
+            buffer = io.BytesIO()
+            plt.savefig(buffer, format='png')
+            buffer.seek(0)
+
+            # Encode the BytesIO object as base64
+            base64_encoded_plot = base64.b64encode(buffer.read()).decode('utf-8')
+            print(f"{base64_encoded_plot}")
+            self.histogram = base64_encoded_plot
+            # close plot
+            plt.close()
 
         # The following information should be stored in a database
         attrs = {}
@@ -64,18 +84,11 @@ def gen_data_stats(self, filename, var, nbins=100):
         self.info["Vars"] = list(dict(ds.variables).keys())
         self.info["Vstats"] = vstats
         self.info["Mstats"] = get_stats(mratio)
-        # self.info["Hist"] = hist
-        print(self.info)
+        # print(self.info)
     
     def write_json(self):
         outfile = self.output_dir / "info.txt"
         with open(outfile.as_posix(), "w") as f:
             yaml.dump(self.info, f)
         return outfile
-    
-    def write_png(self):
-        outfile = self.output_dir / "histime.png"
-        return outfile
-
-
-
+    
\ No newline at end of file
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 8b2c03a..8cc2da1 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -182,6 +182,7 @@ def _handler(self, request, response):
                 "mean": stats["mean"],
                 "stddev": stats["std"],
                 "info": json.dumps(datastats.info, separators=(',', ':')),
+                "histogram": datastats.histogram,
             }, 
             [datasets[0].as_posix()], 
             [f"{datasets[0].as_posix()}_infilled.nc"],
diff --git a/duck/processes/wps_dashboard.py b/duck/processes/wps_dashboard.py
index 784fa1d..98d1275 100644
--- a/duck/processes/wps_dashboard.py
+++ b/duck/processes/wps_dashboard.py
@@ -53,7 +53,7 @@ def _handler(self, request, response):
     
     def write_html(self, df, workdir):
         # Convert the DataFrame to an HTML table
-        html_table = df.to_html(index=False)
+        html_table = df.to_html(escape=False, index=False)
 
         # Define the HTML template
         html_template = f"""
diff --git a/duck/provenance.py b/duck/provenance.py
index 5200fac..63243ac 100644
--- a/duck/provenance.py
+++ b/duck/provenance.py
@@ -95,6 +95,7 @@ def add_operator(self, operator, parameters, collection, output, start, end):
             "stddev",
             "mean",
             "info",
+            "histogram",
             "dataset_name",
             "variable_name",
         ]:
diff --git a/duck/query.py b/duck/query.py
index f0f953d..e9e3d8f 100644
--- a/duck/query.py
+++ b/duck/query.py
@@ -1,10 +1,22 @@
 from duck.db import GraphDB
 import pandas as pd
+import base64
+from PIL import Image
+import io
+import json
 
+def display_image(base64_image):
+    img_data = base64.b64decode(base64_image)
+    img = Image.open(io.BytesIO(img_data))
+    return '<img src="data:image/png;base64,{}" width="200"/>'.format(base64_image)
+
+def display_json(data):
+    content = json.loads(data)
+    return f"<pre>{content}</pre>"
 
 def query():
     query_str = """
-        SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?min ?max ?mean ?stddev ?info
+        SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?min ?max ?mean ?stddev ?info ?histogram
         WHERE {
             ?exec rdf:type provone:Execution ;
                 rdfs:label ?process ;
@@ -16,7 +28,8 @@ def query():
                 clint:max ?max ;
                 clint:mean ?mean ;
                 clint:stddev ?stddev ;
-                clint:info ?info .
+                clint:info ?info ;
+                clint:histogram ?histogram .
 
             ?input rdf:type prov:Entity .
         
@@ -43,8 +56,9 @@ def query():
         max = row.max.value
         mean = row.mean.value
         stddev = row.stddev.value
-        info = row.info.value
-        data.append({
+        info = json.loads(row.info.value)
+        histogram = row.histogram.value
+        entry = {
             "Process": process, 
             "Dataset": dataset,
             "Variable": variable, 
@@ -52,11 +66,14 @@ def query():
             "End Time": end_time,
             "Input": input,
             "Output": output,
-            "Min": min,
-            "Max": max,
-            "Mean": mean,
-            "StdDev": stddev,
-            "Info": info,
-        })
+            # "Min": min,
+            # "Max": max,
+            # "Mean": mean,
+            # "StdDev": stddev,
+            "Histogram": display_image(histogram),
+        }
+        for key in info:
+            entry[key] = info[key]
+        data.append(entry)
     df = pd.DataFrame(data)
     return df

From 0d3b3a7586b220fd173f1f9fa77a2f1211911264 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 14:15:43 +0200
Subject: [PATCH 24/49] update tests

---
 tests/test_wps_caps.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_wps_caps.py b/tests/test_wps_caps.py
index 04c1acb..6d0f9ba 100644
--- a/tests/test_wps_caps.py
+++ b/tests/test_wps_caps.py
@@ -13,4 +13,5 @@ def test_wps_caps():
                             '/ows:Identifier')
     assert sorted(names.split()) == [
         'crai',
+        'dashboard',
     ]

From 12a5e84dc264120cef2693963be2fc2eb30d22cf Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 14:26:51 +0200
Subject: [PATCH 25/49] update report

---
 .gitignore    | 4 ++--
 duck/query.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 167a59d..23be439 100644
--- a/.gitignore
+++ b/.gitignore
@@ -112,5 +112,5 @@ testdata.json
 # macos
 .DS_Store
 
-# provenance
-provenance.db
+# tests
+PET0.ESMF_LogFile
diff --git a/duck/query.py b/duck/query.py
index e9e3d8f..dc94a45 100644
--- a/duck/query.py
+++ b/duck/query.py
@@ -11,7 +11,7 @@ def display_image(base64_image):
     return '<img src="data:image/png;base64,{}" width="200"/>'.format(base64_image)
 
 def display_json(data):
-    content = json.loads(data)
+    content = json.dumps(data, indent=4)
     return f"<pre>{content}</pre>"
 
 def query():
@@ -73,7 +73,7 @@ def query():
             "Histogram": display_image(histogram),
         }
         for key in info:
-            entry[key] = info[key]
+            entry[key] = display_json(info[key])
         data.append(entry)
     df = pd.DataFrame(data)
     return df

From 17d40c1c41ad5af7d5b66bbf7c200db4cf3fa57c Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 14:33:53 +0200
Subject: [PATCH 26/49] use yaml output

---
 duck/query.py    | 3 ++-
 environment.yml  | 1 +
 requirements.txt | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/duck/query.py b/duck/query.py
index dc94a45..a634227 100644
--- a/duck/query.py
+++ b/duck/query.py
@@ -4,6 +4,7 @@
 from PIL import Image
 import io
 import json
+import yaml
 
 def display_image(base64_image):
     img_data = base64.b64decode(base64_image)
@@ -11,7 +12,7 @@ def display_image(base64_image):
     return '<img src="data:image/png;base64,{}" width="200"/>'.format(base64_image)
 
 def display_json(data):
-    content = json.dumps(data, indent=4)
+    content = yaml.dump(data, default_flow_style=True, indent=2)
     return f"<pre>{content}</pre>"
 
 def query():
diff --git a/environment.yml b/environment.yml
index 3289096..9abd233 100644
--- a/environment.yml
+++ b/environment.yml
@@ -37,3 +37,4 @@ dependencies:
 - rdflib-sqlalchemy
 - sqlalchemy<2
 - pandas
+- pyyaml
diff --git a/requirements.txt b/requirements.txt
index 246da52..f3f6015 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,3 +13,4 @@ rdflib
 rdflib-sqlalchemy
 sqlalchemy<2
 pandas
+pyyaml

From ed0aaf812059110fe6a0d9e06358e9dd0a008df8 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 14:41:34 +0200
Subject: [PATCH 27/49] fix tests

---
 tests/test_wps_clintai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_wps_clintai.py b/tests/test_wps_clintai.py
index cf24782..e6e355d 100644
--- a/tests/test_wps_clintai.py
+++ b/tests/test_wps_clintai.py
@@ -15,7 +15,7 @@
 from duck.processes.wps_clintai import ClintAI
 
 
-@pytest.mark.online
+@pytest.mark.skip
 def test_wps_clintai_hadcrut4_temperature_anomaly():
     client = client_for(Service(processes=[ClintAI()]))
     datainputs = f"file=@xlink:href={HADCRUT4_TAS_NC_ZIP};variable_name=temperature_anomaly"

From 68970df54f613cfbf09979c915cf155f3d83ff5e Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 15:22:05 +0200
Subject: [PATCH 28/49] pep8

---
 duck/data_stats.py              |  2 --
 duck/db.py                      |  1 +
 duck/processes/wps_clintai.py   | 23 ++++-------------------
 duck/processes/wps_dashboard.py |  1 +
 duck/provenance.py              |  1 -
 duck/query.py                   | 18 +++++++++---------
 6 files changed, 15 insertions(+), 31 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index fd24d8b..2edd7ec 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -6,8 +6,6 @@
 import io
 import base64
 
-# import matplotlib
-# matplotlib.use('agg')
 
 def get_stats(data):
     return {
diff --git a/duck/db.py b/duck/db.py
index fb9accd..9c429bc 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -4,6 +4,7 @@
 # Provide the path to the SQLite database in the local folder
 DB_URL = "sqlite:///provenance.sqlite"
 
+
 class GraphDB(object):
     def __init__(self):
         # Create a graph with a specific backend store
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 8cc2da1..d3a96a3 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -5,9 +5,6 @@
 import json
 import xarray as xr
 
-import matplotlib
-matplotlib.use('agg')
-
 from pywps import Process
 from pywps import LiteralInput, ComplexInput, ComplexOutput
 from pywps import FORMATS, Format
@@ -20,6 +17,10 @@
 from duck.data_stats import DataStats
 
 import logging
+
+import matplotlib
+matplotlib.use('agg')
+
 LOGGER = logging.getLogger("PYWPS")
 
 FORMAT_PNG = Format("image/png", extension=".png", encoding="base64")
@@ -29,18 +30,6 @@
 models_list = list(craimodels.info_models().keys())
 
 
-def dummy(workdir, datasets):
-    outputs_path =  workdir / "outputs" 
-    outputs_path.mkdir()
-    outfile = workdir / "outputs" / str(datasets[0].stem+"_infilled.nc")
-    print(outfile)
-    with outfile.open(mode='w') as file:
-        file.write("dummy\n")
-    plotfile = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
-    print(plotfile)
-    with plotfile.open(mode='w') as file:
-        file.write("dummy plot\n")
-
 class ClintAI(Process):
     def __init__(self):
         inputs = [
@@ -135,10 +124,6 @@ def _handler(self, request, response):
                 raise ProcessError("Could not find variable {} in {}.".format(variable_name, dataset))
 
             try:
-                # dummy start
-                # dummy(workdir, datasets)
-                # end
-
                 clintai.run(
                     dataset,
                     dataset_name=dataset_name,
diff --git a/duck/processes/wps_dashboard.py b/duck/processes/wps_dashboard.py
index 98d1275..156bde4 100644
--- a/duck/processes/wps_dashboard.py
+++ b/duck/processes/wps_dashboard.py
@@ -4,6 +4,7 @@
 
 from duck import query
 
+
 class Dashboard(Process):
     def __init__(self):
         inputs = [
diff --git a/duck/provenance.py b/duck/provenance.py
index 63243ac..25b1a1a 100644
--- a/duck/provenance.py
+++ b/duck/provenance.py
@@ -1,7 +1,6 @@
 import os
 import uuid
 import json
-from datetime import datetime
 import pathlib
 
 from prov.identifier import Namespace
diff --git a/duck/query.py b/duck/query.py
index a634227..fb41a01 100644
--- a/duck/query.py
+++ b/duck/query.py
@@ -1,20 +1,20 @@
 from duck.db import GraphDB
 import pandas as pd
-import base64
-from PIL import Image
-import io
 import json
 import yaml
 
+
 def display_image(base64_image):
-    img_data = base64.b64decode(base64_image)
-    img = Image.open(io.BytesIO(img_data))
+    # img_data = base64.b64decode(base64_image)
+    # img = Image.open(io.BytesIO(img_data))
     return '<img src="data:image/png;base64,{}" width="200"/>'.format(base64_image)
 
+
 def display_json(data):
     content = yaml.dump(data, default_flow_style=True, indent=2)
     return f"<pre>{content}</pre>"
 
+
 def query():
     query_str = """
         SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?min ?max ?mean ?stddev ?info ?histogram
@@ -53,10 +53,10 @@ def query():
         input = input.split("urn:clint:")[-1]
         output = row.output.split("/")[-1]
         output = output.split("urn:clint:")[-1]
-        min = row.min.value
-        max = row.max.value
-        mean = row.mean.value
-        stddev = row.stddev.value
+        # min = row.min.value
+        # max = row.max.value
+        # mean = row.mean.value
+        # stddev = row.stddev.value
         info = json.loads(row.info.value)
         histogram = row.histogram.value
         entry = {

From d11bcc065e9bfa5db2acc784b05a0a9ec7411876 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 15:33:07 +0200
Subject: [PATCH 29/49] clean up

---
 duck/data_stats.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index 2edd7ec..2b87861 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -63,7 +63,7 @@ def gen_data_stats(self, filename, var, nbins=100):
 
             # Encode the BytesIO object as base64
             base64_encoded_plot = base64.b64encode(buffer.read()).decode('utf-8')
-            print(f"{base64_encoded_plot}")
+            # print(f"{base64_encoded_plot}")
             self.histogram = base64_encoded_plot
             # close plot
             plt.close()

From d80067399c28515967b8d56643e3ec88620c8e81 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 15:58:57 +0200
Subject: [PATCH 30/49] pep8 with black

---
 duck/data_stats.py              |  26 ++++---
 duck/db.py                      |   8 +-
 duck/processes/wps_clintai.py   | 132 ++++++++++++++++++++------------
 duck/processes/wps_dashboard.py |  22 +++---
 duck/provenance.py              |  15 ++--
 duck/query.py                   |  10 +--
 requirements_dev.txt            |   1 +
 7 files changed, 124 insertions(+), 90 deletions(-)

diff --git a/duck/data_stats.py b/duck/data_stats.py
index 2b87861..1530a25 100644
--- a/duck/data_stats.py
+++ b/duck/data_stats.py
@@ -9,10 +9,10 @@
 
 def get_stats(data):
     return {
-        "min": float(np.nanmin(data)), 
-        "max": float(np.nanmax(data)), 
-        "mean": float(np.nanmean(data)), 
-        "std": float(np.nanstd(data))
+        "min": float(np.nanmin(data)),
+        "max": float(np.nanmax(data)),
+        "mean": float(np.nanmean(data)),
+        "std": float(np.nanstd(data)),
     }
 
 
@@ -25,7 +25,6 @@ def __init__(self, output_dir):
         self.info = None
         self.histogram = None
 
-   
     def gen_data_stats(self, filename, var, nbins=100):
         ds = xr.open_dataset(filename)
 
@@ -47,9 +46,15 @@ def gen_data_stats(self, filename, var, nbins=100):
         # TODO: It would be great to store the distribution graph in a database
         if True:
             plt.close()
-            plt.imshow(hist, aspect="auto", origin='lower', extent=[vstats["min"], vstats["max"], 0, ntime], cmap="gist_ncar")
+            plt.imshow(
+                hist,
+                aspect="auto",
+                origin="lower",
+                extent=[vstats["min"], vstats["max"], 0, ntime],
+                cmap="gist_ncar",
+            )
             ax = plt.gca()
-            ax.grid(color='gray', linestyle='-.', linewidth=1)
+            ax.grid(color="gray", linestyle="-.", linewidth=1)
             plt.xlabel(var)
             plt.ylabel("Timesteps")
             # outfile = self.output_dir / "histogram.png"
@@ -58,11 +63,11 @@ def gen_data_stats(self, filename, var, nbins=100):
             # store as base64
             # Save the plot to a BytesIO object
             buffer = io.BytesIO()
-            plt.savefig(buffer, format='png')
+            plt.savefig(buffer, format="png")
             buffer.seek(0)
 
             # Encode the BytesIO object as base64
-            base64_encoded_plot = base64.b64encode(buffer.read()).decode('utf-8')
+            base64_encoded_plot = base64.b64encode(buffer.read()).decode("utf-8")
             # print(f"{base64_encoded_plot}")
             self.histogram = base64_encoded_plot
             # close plot
@@ -83,10 +88,9 @@ def gen_data_stats(self, filename, var, nbins=100):
         self.info["Vstats"] = vstats
         self.info["Mstats"] = get_stats(mratio)
         # print(self.info)
-    
+
     def write_json(self):
         outfile = self.output_dir / "info.txt"
         with open(outfile.as_posix(), "w") as f:
             yaml.dump(self.info, f)
         return outfile
-    
\ No newline at end of file
diff --git a/duck/db.py b/duck/db.py
index 9c429bc..61b9f4b 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -9,8 +9,8 @@ class GraphDB(object):
     def __init__(self):
         # Create a graph with a specific backend store
         self.graph = Graph(
-            store="SQLAlchemy", 
-            identifier=URIRef("http://example.org/graph"))
+            store="SQLAlchemy", identifier=URIRef("http://example.org/graph")
+        )
         self.graph.open(DB_URL, create=True)
 
     def add(self, data):
@@ -31,10 +31,8 @@ def query(self, query_str):
             "prov": "http://www.w3.org/ns/prov#",
             "provone": "http://purl.dataone.org/provone/2015/01/15/ontology#",
             "dcterms": "http://purl.org/dc/terms/",
-            "clint": "urn:clint:"
+            "clint": "urn:clint:",
         }
         query = prepareQuery(query_str, initNs=namespaces)
         results = self.graph.query(query)
         return results
-
-    
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index d3a96a3..c162846 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -19,7 +19,8 @@
 import logging
 
 import matplotlib
-matplotlib.use('agg')
+
+matplotlib.use("agg")
 
 LOGGER = logging.getLogger("PYWPS")
 
@@ -33,29 +34,46 @@
 class ClintAI(Process):
     def __init__(self):
         inputs = [
-            LiteralInput('dataset_name', 'Dataset name', data_type='string',
-                         abstract='Choose the type of dataset to be infilled.',
-                         allowed_values=models_list,
-                         default=models_list[0]),
-            ComplexInput('file', 'Add your NetCDF file with missing values here',
-                         abstract="Enter a URL pointing to a NetCDF file with missing values.",
-                         min_occurs=1,
-                         max_occurs=1,
-                         default="https://www.metoffice.gov.uk/hadobs/hadcrut5/data/current/non-infilled/HadCRUT.5.0.1.0.anomalies.ensemble_mean.nc", # noqa
-                         supported_formats=[FORMATS.NETCDF, FORMATS.ZIP]),
-            LiteralInput('variable_name', 'Variable name', data_type='string',
-                         abstract='Enter here the variable name to be infilled.',
-                         default='tas_mean'),
+            LiteralInput(
+                "dataset_name",
+                "Dataset name",
+                data_type="string",
+                abstract="Choose the type of dataset to be infilled.",
+                allowed_values=models_list,
+                default=models_list[0],
+            ),
+            ComplexInput(
+                "file",
+                "Add your NetCDF file with missing values here",
+                abstract="Enter a URL pointing to a NetCDF file with missing values.",
+                min_occurs=1,
+                max_occurs=1,
+                default="https://www.metoffice.gov.uk/hadobs/hadcrut5/data/current/non-infilled/HadCRUT.5.0.1.0.anomalies.ensemble_mean.nc",  # noqa
+                supported_formats=[FORMATS.NETCDF, FORMATS.ZIP],
+            ),
+            LiteralInput(
+                "variable_name",
+                "Variable name",
+                data_type="string",
+                abstract="Enter here the variable name to be infilled.",
+                default="tas_mean",
+            ),
         ]
         outputs = [
-            ComplexOutput('output', 'Reconstructed dataset',
-                          abstract='NetCDF output produced by CRAI.',
-                          as_reference=True,
-                          supported_formats=[FORMATS.NETCDF]),
-            ComplexOutput('plot', 'Preview of the first time step',
-                          # abstract='Plot of original input file. First timestep.',
-                          as_reference=True,
-                          supported_formats=[FORMAT_PNG]),
+            ComplexOutput(
+                "output",
+                "Reconstructed dataset",
+                abstract="NetCDF output produced by CRAI.",
+                as_reference=True,
+                supported_formats=[FORMATS.NETCDF],
+            ),
+            ComplexOutput(
+                "plot",
+                "Preview of the first time step",
+                # abstract='Plot of original input file. First timestep.',
+                as_reference=True,
+                supported_formats=[FORMAT_PNG],
+            ),
         ]
 
         super(ClintAI, self).__init__(
@@ -68,14 +86,21 @@ def __init__(self):
                 Metadata(
                     title="CRAI Logo",
                     href="https://github.com/FREVA-CLINT/duck/raw/main/docs/source/_static/crai_logo.png",
-                    role=MEDIA_ROLE),
-                Metadata('CRAI', 'https://github.com/FREVA-CLINT/climatereconstructionAI'),
-                Metadata('Clint Project', 'https://climateintelligence.eu/'),
-                Metadata('HadCRUT on Wikipedia', 'https://en.wikipedia.org/wiki/HadCRUT'),
-                Metadata('HadCRUT4', 'https://www.metoffice.gov.uk/hadobs/hadcrut4/'),
-                Metadata('HadCRUT5', 'https://www.metoffice.gov.uk/hadobs/hadcrut5/'),
-                Metadata('Near Surface Air Temperature',
-                         'https://www.atlas.impact2c.eu/en/climate/temperature/?parent_id=22'),
+                    role=MEDIA_ROLE,
+                ),
+                Metadata(
+                    "CRAI", "https://github.com/FREVA-CLINT/climatereconstructionAI"
+                ),
+                Metadata("Clint Project", "https://climateintelligence.eu/"),
+                Metadata(
+                    "HadCRUT on Wikipedia", "https://en.wikipedia.org/wiki/HadCRUT"
+                ),
+                Metadata("HadCRUT4", "https://www.metoffice.gov.uk/hadobs/hadcrut4/"),
+                Metadata("HadCRUT5", "https://www.metoffice.gov.uk/hadobs/hadcrut5/"),
+                Metadata(
+                    "Near Surface Air Temperature",
+                    "https://www.atlas.impact2c.eu/en/climate/temperature/?parent_id=22",
+                ),
             ],
             inputs=inputs,
             outputs=outputs,
@@ -86,42 +111,43 @@ def __init__(self):
     def _handler(self, request, response):
         start_time = datetime.now().isoformat(timespec="seconds")
 
-        dataset_name = request.inputs['dataset_name'][0].data
+        dataset_name = request.inputs["dataset_name"][0].data
         print(f"{dataset_name}")
-        file = request.inputs['file'][0].file
+        file = request.inputs["file"][0].file
         print(f"{file}")
-        variable_name = request.inputs['variable_name'][0].data
+        variable_name = request.inputs["variable_name"][0].data
         print(f"{variable_name}")
 
-        response.update_status('Prepare dataset ...', 0)
+        response.update_status("Prepare dataset ...", 0)
         workdir = Path(self.workdir)
-       
+
         zipfile = False
         if Path(file).suffix == ".zip":
-            with ZipFile(file, 'r') as zip:
+            with ZipFile(file, "r") as zip:
                 print("extraction zip file", workdir)
                 zip.extractall(workdir.as_posix())
                 zipfile = True
 
         try:
-            datasets = sorted(workdir.rglob('*.nc'), key=os.path.getmtime)
+            datasets = sorted(workdir.rglob("*.nc"), key=os.path.getmtime)
         except Exception:
             raise ProcessError("Could not extract netcdf file.")
 
         ndata = len(datasets)
         if ndata == 0:
             raise ProcessError("Could not find netcdf files.")
-        
+
         # run crai
         istep = 100 / ndata
         i = 0
         for dataset in datasets:
-
             ds = xr.open_dataset(dataset)
 
             vars = list(ds.keys())
             if variable_name not in vars:
-                raise ProcessError("Could not find variable {} in {}.".format(variable_name, dataset))
+                raise ProcessError(
+                    "Could not find variable {} in {}.".format(variable_name, dataset)
+                )
 
             try:
                 clintai.run(
@@ -129,7 +155,8 @@ def _handler(self, request, response):
                     dataset_name=dataset_name,
                     variable_name=variable_name,
                     outdir=workdir,
-                    update_status=[response.update_status, i, istep])
+                    update_status=[response.update_status, i, istep],
+                )
             except Exception as e:
                 raise ProcessError(str(e))
 
@@ -138,16 +165,20 @@ def _handler(self, request, response):
         if zipfile:
             outfile = ".".join(file.split(".")[:-1]) + "_infilled.zip"
             outfile = workdir / "outputs" / outfile
-            infiles = sorted((workdir / "outputs").rglob('*_infilled.nc'), key=os.path.getmtime)
+            infiles = sorted(
+                (workdir / "outputs").rglob("*_infilled.nc"), key=os.path.getmtime
+            )
 
-            with ZipFile(outfile, 'w') as zip:
+            with ZipFile(outfile, "w") as zip:
                 for infile in infiles:
                     zip.write(infile, arcname=infile.as_posix().split("/")[-1])
         else:
-            outfile = workdir / "outputs" / str(datasets[0].stem+"_infilled.nc")
+            outfile = workdir / "outputs" / str(datasets[0].stem + "_infilled.nc")
 
         response.outputs["output"].file = outfile
-        response.outputs["plot"].file = workdir / "outputs" / str(datasets[0].stem+"_combined.1_0.png")
+        response.outputs["plot"].file = (
+            workdir / "outputs" / str(datasets[0].stem + "_combined.1_0.png")
+        )
 
         # stats
         datastats = DataStats(self.workdir)
@@ -158,7 +189,7 @@ def _handler(self, request, response):
         end_time = datetime.now().isoformat(timespec="seconds")
         prov = Provenance(self.workdir)
         prov.add_operator(
-            "crai", 
+            "crai",
             {
                 "dataset_name": dataset_name,
                 "variable_name": variable_name,
@@ -166,10 +197,10 @@ def _handler(self, request, response):
                 "max": stats["max"],
                 "mean": stats["mean"],
                 "stddev": stats["std"],
-                "info": json.dumps(datastats.info, separators=(',', ':')),
+                "info": json.dumps(datastats.info, separators=(",", ":")),
                 "histogram": datastats.histogram,
-            }, 
-            [datasets[0].as_posix()], 
+            },
+            [datasets[0].as_posix()],
             [f"{datasets[0].as_posix()}_infilled.nc"],
             start_time,
             end_time,
@@ -177,6 +208,5 @@ def _handler(self, request, response):
         prov.store_rdf()
         # prov end
 
-
-        response.update_status('done.', 100)
+        response.update_status("done.", 100)
         return response
diff --git a/duck/processes/wps_dashboard.py b/duck/processes/wps_dashboard.py
index 156bde4..10b5b08 100644
--- a/duck/processes/wps_dashboard.py
+++ b/duck/processes/wps_dashboard.py
@@ -20,19 +20,19 @@ def __init__(self):
         ]
         outputs = [
             ComplexOutput(
-                'report', 
-                'Generated HTML Report', 
+                "report",
+                "Generated HTML Report",
                 as_reference=True,
-                supported_formats=[Format('text/html')]
+                supported_formats=[Format("text/html")],
             ),
         ]
 
         super(Dashboard, self).__init__(
             self._handler,
-            identifier='dashboard',
-            title='Generate HTML Report',
-            version='1.0',
-            abstract='Generate an HTML report from a provenance database.',
+            identifier="dashboard",
+            title="Generate HTML Report",
+            version="1.0",
+            abstract="Generate an HTML report from a provenance database.",
             inputs=inputs,
             outputs=outputs,
         )
@@ -47,11 +47,11 @@ def _handler(self, request, response):
         # Generate an HTML report from the DataFrame
         html_report = self.write_html(df, workdir)
 
-        response.outputs['report'].file = html_report
-        response.outputs['report'].output_format = Format('text/html')
+        response.outputs["report"].file = html_report
+        response.outputs["report"].output_format = Format("text/html")
 
         return response
-    
+
     def write_html(self, df, workdir):
         # Convert the DataFrame to an HTML table
         html_table = df.to_html(escape=False, index=False)
@@ -92,7 +92,7 @@ def write_html(self, df, workdir):
 
         # Write the HTML template to a file
         outfile = workdir / "provenance_report.html"
-        with outfile.open(mode='w') as file:
+        with outfile.open(mode="w") as file:
             file.write(html_template)
 
         return outfile
diff --git a/duck/provenance.py b/duck/provenance.py
index 25b1a1a..8953de8 100644
--- a/duck/provenance.py
+++ b/duck/provenance.py
@@ -108,7 +108,7 @@ def add_operator(self, operator, parameters, collection, output, start, end):
             label=operator,
             attributes=attributes,
             start=start,
-            end=end
+            end=end,
         )
         # input data
         ds_in = os.path.basename(collection[0])
@@ -136,7 +136,9 @@ def _data_entitiy(self, identifier, label=None, value=None):
             )
         return entity
 
-    def _execution_activity(self, identifier, label=None, attributes=None, start=None, end=None):
+    def _execution_activity(
+        self, identifier, label=None, attributes=None, start=None, end=None
+    ):
         records = self.doc.get_record(identifier)
         if records:
             activity = records[0]
@@ -151,9 +153,9 @@ def _execution_activity(self, identifier, label=None, attributes=None, start=Non
         if attributes:
             activity.add_attributes(attributes)
         if start and end:
-            time_attribures={
+            time_attribures = {
                 prov.PROV_ATTR_STARTTIME: start,
-                prov.PROV_ATTR_ENDTIME: end
+                prov.PROV_ATTR_ENDTIME: end,
             }
             activity.add_attributes(time_attribures)
         return activity
@@ -171,12 +173,12 @@ def write_png(self):
 
     def get_provn(self):
         return self.doc.get_provn()
-    
+
     def get_rdf(self):
         # Serialize the PROV document to RDF format
         rdf_data = self.doc.serialize(format="rdf", rdf_format="turtle")
         return rdf_data
-    
+
     def write_rdf(self):
         outfile = self.output_dir / "provenance.ttl"
         with open(outfile.as_posix(), "w") as f:
@@ -191,4 +193,3 @@ def dump_json(self):
 
     def json(self):
         return json.loads(self.dump_json())
-    
\ No newline at end of file
diff --git a/duck/query.py b/duck/query.py
index fb41a01..ff11927 100644
--- a/duck/query.py
+++ b/duck/query.py
@@ -33,11 +33,11 @@ def query():
                 clint:histogram ?histogram .
 
             ?input rdf:type prov:Entity .
-        
+
             ?output rdf:type prov:Entity ;
                 prov:qualifiedDerivation [ prov:entity ?input; prov:hadActivity ?exec ] .
     }
-    """
+    """  # noqa
     graph_db = GraphDB()
     results = graph_db.query(query_str)
 
@@ -60,10 +60,10 @@ def query():
         info = json.loads(row.info.value)
         histogram = row.histogram.value
         entry = {
-            "Process": process, 
+            "Process": process,
             "Dataset": dataset,
-            "Variable": variable, 
-            "Start Time": start_time, 
+            "Variable": variable,
+            "Start Time": start_time,
             "End Time": end_time,
             "Input": input,
             "Output": output,
diff --git a/requirements_dev.txt b/requirements_dev.txt
index dd75b60..d5c53e0 100644
--- a/requirements_dev.txt
+++ b/requirements_dev.txt
@@ -15,3 +15,4 @@ Click
 cruft
 # Changing dependencies above this comment will create merge conflicts when updating the cookiecutter template with cruft. Add extra requirements below this line.
 requests
+black

From 60247db93301939c537507952f9092f16debd76b Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Tue, 5 Sep 2023 18:20:58 +0200
Subject: [PATCH 31/49] clean up

---
 duck/processes/wps_clintai.py |   6 +
 duck/query.py                 |   6 +-
 spec-list.txt                 | 513 +++++++++++++++++++---------------
 3 files changed, 293 insertions(+), 232 deletions(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index c162846..4b609c1 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -27,6 +27,7 @@
 FORMAT_PNG = Format("image/png", extension=".png", encoding="base64")
 
 MEDIA_ROLE = "http://www.opengis.net/spec/wps/2.0/def/process/description/media"
+INFO_ROLE = "https://clint.dkrz.de/spec/crai/info"
 
 models_list = list(craimodels.info_models().keys())
 
@@ -83,6 +84,11 @@ def __init__(self):
             version="0.1.0",
             abstract="AI-enhanced climate service to infill missing values in climate datasets.",
             metadata=[
+                Metadata(
+                    title="CRAI Info Model",
+                    href='{"test":1}',
+                    role=INFO_ROLE,
+                ),
                 Metadata(
                     title="CRAI Logo",
                     href="https://github.com/FREVA-CLINT/duck/raw/main/docs/source/_static/crai_logo.png",
diff --git a/duck/query.py b/duck/query.py
index ff11927..4876937 100644
--- a/duck/query.py
+++ b/duck/query.py
@@ -17,7 +17,7 @@ def display_json(data):
 
 def query():
     query_str = """
-        SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?min ?max ?mean ?stddev ?info ?histogram
+        SELECT ?process ?dataset ?variable ?startTime ?endTime ?input ?output ?info ?histogram
         WHERE {
             ?exec rdf:type provone:Execution ;
                 rdfs:label ?process ;
@@ -25,10 +25,6 @@ def query():
                 clint:variable_name ?variable ;
                 prov:startedAtTime ?startTime ;
                 prov:endedAtTime ?endTime ;
-                clint:min ?min ;
-                clint:max ?max ;
-                clint:mean ?mean ;
-                clint:stddev ?stddev ;
                 clint:info ?info ;
                 clint:histogram ?histogram .
 
diff --git a/spec-list.txt b/spec-list.txt
index a74bda2..3c200da 100644
--- a/spec-list.txt
+++ b/spec-list.txt
@@ -3,331 +3,390 @@
 # platform: linux-64
 @EXPLICIT
 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.12.7-ha878542_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2023.7.22-hbcca054_0.conda
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-hab24e00_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-12.2.0-h337968e_19.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-12.2.0-h46fd767_19.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.1.0-h15d22d2_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.1.0-hfd8a6a1_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/mpi-1.0-mpich.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.10-3_cp310.conda
-https://conda.anaconda.org/conda-forge/noarch/tzdata-2022g-h191b570_0.conda
+https://conda.anaconda.org/conda-forge/noarch/tzdata-2023c-h71feb2d_0.conda
 https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-12.2.0-h69a702a_19.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.1.0-h69a702a_0.conda
 https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_kmp_llvm.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-12.2.0-h65d4601_19.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.8-h166bdaf_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.1.0-he5830b7_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.9-hd590300_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/aom-3.5.0-h27087fc_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/attr-2.5.1-h166bdaf_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.0-hd590300_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.18.1-h7f98852_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/charls-2.4.1-hcb278e6_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/dav1d-1.0.0-h166bdaf_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/expat-2.5.0-h27087fc_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.10-nompi_hf0379b8_106.conda
+https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.19.1-hd590300_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/charls-2.4.2-h59595ed_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/dav1d-1.2.1-hd590300_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/freexl-1.0.6-h166bdaf_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/geos-3.11.1-h27087fc_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.10-h36c2ea0_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/geos-3.12.0-h59595ed_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/gettext-0.21.1-h27087fc_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.1-h36c2ea0_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/gflags-2.2.2-he1b5a44_1004.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.1-h0b41bf4_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/gmp-6.2.1-h58526e2_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h58526e2_1001.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/gstreamer-orc-0.4.33-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/icu-70.1-h27087fc_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h166bdaf_2.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/json-c-0.16-hc379101_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/icu-72.1-hcb278e6_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/json-c-0.17-h7ab15ed_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/jxrlib-1.1-h7f98852_2.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/lame-3.100-h166bdaf_1003.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libabseil-20220623.0-cxx17_h05df665_6.conda
+https://conda.anaconda.org/conda-forge/linux-64/libabseil-20230125.3-cxx17_h59595ed_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-hcb278e6_1.conda
-https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.0.9-h166bdaf_8.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libdb-6.2.32-h9c3ff4c_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.17-h0b41bf4_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.0.9-h166bdaf_9.conda
+https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.18-h0b41bf4_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.5.0-hcb278e6_1.conda
 https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-2.1.5.1-h0b41bf4_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.0-h7f98852_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libnuma-2.0.16-h0b41bf4_1.conda
 https://conda.anaconda.org/conda-forge/linux-64/libogg-1.3.4-h7f98852_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.21-pthreads_h78a6416_3.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.23-pthreads_h80387f5_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libopus-1.3.1-h7f98852_1.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/libtool-2.4.7-h27087fc_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libudev1-252-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.32.1-h7f98852_1000.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.2.4-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-h166bdaf_4.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libutf8proc-2.8.0-h166bdaf_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libuv-1.46.0-hd590300_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.3.1-hd590300_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda
 https://conda.anaconda.org/conda-forge/linux-64/libzopfli-1.0.3-h9c3ff4c_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/mpg123-1.31.2-hcb278e6_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.3-h27087fc_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-h516909a_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/mpg123-1.31.3-hcb278e6_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/mpich-4.1.2-h846660c_100.conda
+https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-hcb278e6_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/openssl-3.0.8-h0b41bf4_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.2-hd590300_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/pixman-0.40.0-h36c2ea0_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/re2-2023.02.01-hcb278e6_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/rdma-core-28.9-h59595ed_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/re2-2023.03.02-h8c504da_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/sleef-3.5.1-h9b69904_2.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.9-hbd366e4_2.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/tzcode-2022g-h166bdaf_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.10-h9fff704_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/tzcode-2023c-h0b41bf4_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/xorg-kbproto-1.0.7-h7f98852_1002.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.0.10-h7f98852_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.9-h7f98852_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.1-hd590300_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.11-hd590300_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.3-h7f98852_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-renderproto-0.11.1-h7f98852_1002.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xorg-xextproto-7.3.0-h7f98852_1002.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/xorg-xextproto-7.3.0-h0b41bf4_1003.conda
+https://conda.anaconda.org/conda-forge/linux-64/xorg-xf86vidmodeproto-2.3.1-h7f98852_1002.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/zfp-1.0.0-h27087fc_3.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/zlib-ng-2.0.6-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/jack-1.9.22-h11f4161_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libavif-0.11.1-h5cdd6b5_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-16_linux64_openblas.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.0.9-h166bdaf_8.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.0.9-h166bdaf_8.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libcap-2.66-ha37c62d_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/zlib-ng-2.0.7-h0b41bf4_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.2-hc309b26_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.2.17-h4d4d85c_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.12-h4d4d85c_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.17-h4d4d85c_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/expat-2.5.0-hcb278e6_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/glog-0.6.0-h6f12383_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h501b40f_6.conda
+https://conda.anaconda.org/conda-forge/linux-64/libavif-0.11.1-h8182462_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-17_linux64_openblas.conda
+https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.0.9-h166bdaf_9.conda
+https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.0.9-h166bdaf_9.conda
+https://conda.anaconda.org/conda-forge/linux-64/libcap-2.69-h0f662aa_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.10-h28343ad_4.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libflac-1.4.2-h27087fc_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libgpg-error-1.46-h620e276_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libllvm11-11.1.0-he0ac6c6_5.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.51.0-hff17c54_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libflac-1.4.3-h59595ed_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libgpg-error-1.47-h71f35ed_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libllvm14-14.0.6-hcd5def8_4.conda
+https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.52.0-h61bc06f_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libpnetcdf-1.12.3-mpi_mpich_h4ac58e2_101.conda
 https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.39-h753d276_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-3.21.12-h3eb15da_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-ha49c73b_12.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.40.0-h753d276_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.10.0-hf14f497_3.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-4.23.3-hd1fb520_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-hb58d41b_14.conda
+https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.43.0-h2797004_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.0-h0841786_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libvorbis-1.3.7-h9c3ff4c_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.13-h7f98852_1004.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.10.3-h7463322_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libzip-1.9.2-hc929e4a_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/llvm-openmp-15.0.7-h0cdce71_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/mysql-common-8.0.32-ha901b37_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.15-h0b41bf4_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.11.5-h0d562d8_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libzip-1.10.1-h2629f0a_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/mpfr-4.2.0-hb012696_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/mysql-common-8.0.33-hf1915f5_2.conda
 https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.40-hc3806b6_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/readline-8.1.2-h0f457ee_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/s2n-1.3.49-h06160fa_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.12-h27826a3_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.3-hd9c2040_1000.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h3eb15da_6.conda
-https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.3-hafa529b_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.0.9-h166bdaf_8.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/c-blosc2-2.6.1-hf91038e_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/ucx-1.14.1-h64cca9d_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.4-h7391055_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-hd590300_5.conda
+https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.5-hfc55251_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.13.32-h019f825_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.5-h0f2a231_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h6582d0a_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.0.9-h166bdaf_9.conda
+https://conda.anaconda.org/conda-forge/linux-64/c-blosc2-2.10.2-hb4ffafa_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_1.conda
-https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h9772cbc_5.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/krb5-1.20.1-h81ceb04_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-16_linux64_openblas.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.2-h659d440_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.6.2-h039dbb9_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-17_linux64_openblas.conda
 https://conda.anaconda.org/conda-forge/linux-64/libgcrypt-1.10.1-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libglib-2.74.1-h606061b_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.51.1-h4fad500_1.conda
-https://conda.anaconda.org/conda-forge/linux-64/libhwloc-2.8.0-h32351e8_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-16_linux64_openblas.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libllvm15-15.0.7-hadd5161_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libsndfile-1.2.0-hb75c966_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.0-h6adf6a1_2.conda
-https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.0.3-he3ba5ed_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h873f0b0_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/mysql-libs-8.0.32-hd7da12d_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/nss-3.82-he02c5a1_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/python-3.10.9-he550d4f_0_cpython.conda
-https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.40.0-h4ff8645_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xcb-util-0.4.0-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xcb-util-keysyms-0.4.0-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.9-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.1-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.7.2-h7f98852_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libglib-2.76.4-hebfc3b9_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.56.2-h3905398_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libhwloc-2.9.2-default_h554bfaf_1009.conda
+https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-17_linux64_openblas.conda
+https://conda.anaconda.org/conda-forge/linux-64/libllvm15-15.0.7-h5cf9203_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/libsndfile-1.2.2-hbc2eb40_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.18.1-h8fd135c_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.1-h8b53f26_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h0054252_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/llvm-openmp-16.0.6-h4dfa4b3_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/mpc-1.3.1-hfe3b2da_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/mysql-libs-8.0.33-hca2cd23_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/nss-3.92-h1d7d5a4_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/orc-1.9.0-h385abfd_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/python-3.10.12-hd12c33a_0_cpython.conda
+https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.43.0-h2c6b66d_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/xcb-util-0.4.0-hd590300_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/xcb-util-keysyms-0.4.0-h8ee46fc_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.9-hd590300_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.1-h8ee46fc_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.6-h8ee46fc_0.conda
 https://conda.anaconda.org/conda-forge/noarch/absl-py-1.4.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.13-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/appdirs-1.4.4-pyh9f0ad1d_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/asciitree-0.3.3-py_2.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/attrs-22.2.0-pyh71513ae_0.conda
-https://conda.anaconda.org/conda-forge/noarch/blinker-1.5-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/brotli-1.0.9-h166bdaf_8.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/cachetools-5.3.0-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/certifi-2022.12.7-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-2.1.1-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/click-8.1.3-unix_pyhd8ed1ab_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-hd4edc92_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/attrs-23.1.0-pyh71513ae_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.3.2-h2e3709c_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.7.12-hc865f51_1.conda
+https://conda.anaconda.org/conda-forge/noarch/blinker-1.6.2-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/brotli-1.0.9-h166bdaf_9.conda
+https://conda.anaconda.org/conda-forge/noarch/cachetools-5.3.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/certifi-2023.7.22-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.2.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda
 https://conda.anaconda.org/conda-forge/noarch/cloudpickle-2.2.1-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/cycler-0.11.0-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/dataclasses-0.8-pyhc8e2a94_3.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/dbus-1.13.6-h5008d03_3.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/docutils-0.19-py310hff52083_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py310hff52083_0.conda
 https://conda.anaconda.org/conda-forge/noarch/entrypoints-0.4-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.3-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/fasteners-0.17.3-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/filelock-3.12.3-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.2-h14ed4e7_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/frozenlist-1.3.3-py310h5764c6d_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.1.0-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/glib-tools-2.74.1-h6239696_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/greenlet-2.0.2-py310heca2aa9_0.conda
-https://conda.anaconda.org/conda-forge/noarch/heapdict-1.0.1-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/frozenlist-1.4.0-py310h2372a71_0.conda
+https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.9.0-pyh1a96a4e_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.10-h6b639ba_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/glib-tools-2.76.4-hfc55251_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/gmpy2-2.1.2-py310h3ec546c_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/greenlet-2.0.2-py310hc6cd4ac_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/grpcio-1.56.2-py310h1b8f574_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h977cf35_4.conda
+https://conda.anaconda.org/conda-forge/noarch/humanize-4.8.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.4-py310hbf28c38_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-hfd0df8a_1.conda
-https://conda.anaconda.org/conda-forge/linux-64/libclang13-15.0.7-default_h3e3d535_1.conda
-https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h36d4200_3.conda
-https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.87.0-hdc1c0ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.5-py310hd41b1e2_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.15-haa2dc70_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libclang13-15.0.7-default_h9986a30_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h4637d8d_4.conda
+https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.2.1-hca28451_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-hb675445_3.conda
-https://conda.anaconda.org/conda-forge/linux-64/libsystemd0-252-h2a991cd_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/llvmlite-0.39.1-py310h58363a5_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libpq-15.4-hfc447b1_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libsystemd0-254-h3516f8a_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.3.1-hbf2b3c1_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/llvmlite-0.40.1-py310h1b8f574_0.conda
 https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.2-py310hbdc0903_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.3-py310h9b7343a_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/lz4-4.3.2-py310h0cfdcf0_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.2-py310h1fa729e_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.3-py310h2372a71_0.conda
 https://conda.anaconda.org/conda-forge/noarch/mccabe-0.7.0-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.4-py310hbf28c38_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/mpmath-1.3.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.5-py310hdf3cbec_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/multidict-6.0.4-py310h1fa729e_0.conda
+https://conda.anaconda.org/conda-forge/noarch/munch-4.0.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/numpy-1.23.5-py310h53a5b5f_0.conda
+https://conda.anaconda.org/conda-forge/noarch/networkx-3.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.4-py310ha4c1d20_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda
-https://conda.anaconda.org/conda-forge/noarch/packaging-23.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/packaging-23.1-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/pkgutil-resolve-name-1.3.10-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/pluggy-1.0.0-pyhd8ed1ab_5.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pluggy-1.3.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/ply-3.11-py_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.4-py310h5764c6d_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.5-py310h1fa729e_0.conda
 https://conda.anaconda.org/conda-forge/noarch/pyasn1-0.4.8-py_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/pycodestyle-2.10.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/pycodestyle-2.11.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/pyflakes-3.0.1-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/pyjwt-2.6.0-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pyflakes-3.1.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/pygments-2.16.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/pyjwt-2.8.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.0.9-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/pyrsistent-0.19.3-py310h1fa729e_0.conda
 https://conda.anaconda.org/conda-forge/noarch/pyshp-2.3.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7.1-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0-py310h5764c6d_5.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2023.3-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/pytz-2023.3-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py310h2372a71_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.10.2-py310hcb5633a_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/setuptools-59.5.0-py310hff52083_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-2.2.0-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/sortedcontainers-2.4.0-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-1.0.4-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-1.0.2-py_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.0.1-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-py_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-1.0.3-py_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.5-pyhd8ed1ab_2.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/tbb-2021.7.0-h924138e_1.conda
+https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/tbb-2021.10.0-h00ab1b0_0.conda
 https://conda.anaconda.org/conda-forge/noarch/tblib-1.7.0-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/tensorboard-data-server-0.6.1-py310h600f1e7_4.conda
-https://conda.anaconda.org/conda-forge/noarch/tensorboard-plugin-wit-1.8.1-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/tensorboard-data-server-0.7.0-py310h34c0648_0.conda
 https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/toolz-0.12.0-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/tornado-6.2-py310h5764c6d_1.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.4.0-pyha770c72_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/tornado-6.3.3-py310h2372a71_0.conda
+https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.7.1-pyha770c72_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/unicodedata2-15.0.0-py310h5764c6d_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/wheel-0.38.4-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-h166bdaf_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/zipp-3.12.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/wheel-0.41.2-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-h8ee46fc_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.39-hd590300_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h0b41bf4_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.11-hd590300_0.conda
+https://conda.anaconda.org/conda-forge/noarch/xyzservices-2023.7.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/zict-3.0.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/zipp-3.16.2-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/aiosignal-1.3.1-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/babel-2.11.0-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.7.3-he2921ad_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.9.5-h3a0376c_1.conda
+https://conda.anaconda.org/conda-forge/noarch/babel-2.12.1-pyhd8ed1ab_1.conda
 https://conda.anaconda.org/conda-forge/linux-64/brunsli-0.1-h9c3ff4c_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-hbbf8b49_1016.conda
 https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py310h255011f_3.conda
-https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.2.0-hd9d235c_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.3.0-hbdc6101_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py310hde88566_1.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/cligj-0.7.2-pyhd8ed1ab_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.0.7-py310hdf3cbec_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/curl-7.87.0-hdc1c0ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.0-py310h5764c6d_1.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/flake8-6.0.0-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.38.0-py310h5764c6d_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/glib-2.74.1-h6239696_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/grpcio-1.51.1-py310h4a5735c_1.conda
-https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-nompi_h4df4325_101.conda
-https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-6.0.0-pyha770c72_0.conda
-https://conda.anaconda.org/conda-forge/noarch/importlib_resources-5.10.2-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.1.0-py310hd41b1e2_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.2-py310h2372a71_0.conda
+https://conda.anaconda.org/conda-forge/noarch/flake8-6.1.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.42.1-py310h2372a71_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/glib-2.76.4-hfc55251_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.2-mpi_mpich_ha2c2bf8_0.conda
+https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-6.8.0-pyha770c72_0.conda
+https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.0.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/isodate-0.6.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.2-pyhd8ed1ab_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libclang-15.0.7-default_had23c3d_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libclang-15.0.7-default_h7634d5b_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-hfa28ad5_6.conda
+https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-h840a212_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.5.0-h5d7e998_3.conda
 https://conda.anaconda.org/conda-forge/linux-64/mkl-2022.2.1-h84fe81f_16997.conda
-https://conda.anaconda.org/conda-forge/noarch/munch-2.5.0-py_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/numba-0.56.4-py310ha5257ce_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/numba-0.57.1-py310h0f6aa51_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/numcodecs-0.11.0-py310heca2aa9_1.conda
-https://conda.anaconda.org/conda-forge/noarch/partd-1.3.0-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/pillow-9.4.0-py310h023d228_1.conda
-https://conda.anaconda.org/conda-forge/noarch/pip-23.0-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-h3248436_3.conda
-https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.1-h8ffa02c_2.conda
-https://conda.anaconda.org/conda-forge/linux-64/protobuf-4.21.12-py310heca2aa9_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/pulseaudio-16.1-ha8d29e2_1.conda
+https://conda.anaconda.org/conda-forge/noarch/partd-1.4.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/pillow-10.0.0-py310h582fbeb_0.conda
+https://conda.anaconda.org/conda-forge/noarch/pip-23.2.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.4-h8972f4a_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/proj-9.2.1-ha643af7_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/protobuf-4.23.3-py310hb875b13_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/pulseaudio-client-16.1-hb77b528_4.conda
 https://conda.anaconda.org/conda-forge/noarch/pyasn1-modules-0.2.7-py_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/pygments-2.14.0-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/pytest-7.2.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/pytest-7.4.1-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/pyu2f-0.1.5-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/referencing-0.30.2-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/rsa-4.9-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/shapely-1.8.5-py310h5b266fc_2.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/sip-6.7.7-py310heca2aa9_0.conda
-https://conda.anaconda.org/conda-forge/noarch/tqdm-4.64.1-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.4.0-hd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/werkzeug-2.2.2-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h55805fa_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/yarl-1.8.2-py310h5764c6d_0.conda
-https://conda.anaconda.org/conda-forge/noarch/zict-2.2.0-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/async-timeout-4.0.2-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/bokeh-2.4.3-pyhd8ed1ab_3.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.1-py310h7dcad9a_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/sip-6.7.11-py310hc6cd4ac_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/sqlalchemy-1.4.49-py310h2372a71_0.conda
+https://conda.anaconda.org/conda-forge/noarch/sympy-1.12-pypyh9d50eac_103.conda
+https://conda.anaconda.org/conda-forge/noarch/tqdm-4.66.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.7.1-hd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/werkzeug-2.3.7-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h8d71039_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/yarl-1.9.2-py310h2372a71_0.conda
+https://conda.anaconda.org/conda-forge/noarch/async-timeout-4.0.3-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.3.14-h1678ad6_3.conda
 https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py310h5764c6d_1005.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/cryptography-39.0.1-py310h34c0648_0.conda
-https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.1.1-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h7a142b4_6.conda
-https://conda.anaconda.org/conda-forge/linux-64/gstreamer-1.22.0-h25f0c4b_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-6.0.0-h8e241bc_0.conda
-https://conda.anaconda.org/conda-forge/noarch/humanize-4.6.0-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/imagecodecs-2023.1.23-py310ha3ed6a1_0.conda
-https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.17.3-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.0-ha7026e8_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-nompi_h261ec11_106.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h221c8f1_23.conda
-https://conda.anaconda.org/conda-forge/noarch/markdown-3.4.1-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.6.3-py310he60537e_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/pandas-1.5.3-py310h9b08913_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/poppler-23.01.0-h091648b_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py310h15e2413_1.conda
-https://conda.anaconda.org/conda-forge/linux-64/pyqt5-sip-12.11.0-py310heca2aa9_3.conda
-https://conda.anaconda.org/conda-forge/linux-64/pytorch-1.13.1-cpu_py310hd11e9c7_1.conda
-https://conda.anaconda.org/conda-forge/linux-64/sqlalchemy-2.0.2-py310h1fa729e_0.conda
-https://conda.anaconda.org/conda-forge/noarch/tensorboardx-2.5-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.2-hd532e3d_0.conda
-https://conda.anaconda.org/conda-forge/noarch/zarr-2.13.6-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/aiohttp-3.8.3-py310h5764c6d_1.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/gst-plugins-base-1.22.0-h4243ec0_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.2-h8c90c07_6.conda
-https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.0-nompi_he1eeb6f_102.conda
-https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.2-nompi_py310h55e1e36_100.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cryptography-41.0.3-py310h75e40e8_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h22adcc9_11.conda
+https://conda.anaconda.org/conda-forge/linux-64/gstreamer-1.22.5-h98fc4e7_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-7.3.0-hdb3a94d_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/imagecodecs-2023.8.12-py310hc929067_0.conda
+https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-6.8.0-hd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2023.7.1-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.1-hcd42e92_5.conda
+https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-mpi_mpich_hef9003c_12.conda
+https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h15f6e67_28.conda
+https://conda.anaconda.org/conda-forge/noarch/mako-1.2.4-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/markdown-3.4.4-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.7.2-py310hf38f957_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.0-py310hcc13569_0.conda
+https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.10.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/poppler-23.08.0-hd18248d_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.0-py310h24ef57a_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/pyqt5-sip-12.12.2-py310hc6cd4ac_4.conda
+https://conda.anaconda.org/conda-forge/linux-64/pytorch-2.0.0-cpu_mkl_py310h9f5108b_101.conda
+https://conda.anaconda.org/conda-forge/noarch/rdflib-7.0.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/tensorboardx-2.6.2.2-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.16.3-h84d19f0_1.conda
+https://conda.anaconda.org/conda-forge/noarch/zarr-2.12.0-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/aiohttp-3.8.5-py310h2372a71_0.conda
+https://conda.anaconda.org/conda-forge/noarch/alembic-1.11.3-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.23.0-h40cdbb9_5.conda
+https://conda.anaconda.org/conda-forge/noarch/bokeh-3.2.2-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.22.0-py310h7cbd5c2_0.conda
+https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.9.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/gst-plugins-base-1.22.5-hf7dbed1_0.conda
+https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.19.0-pyhd8ed1ab_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.1-h3d2b3c2_7.conda
+https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.1-mpi_mpich_hdf31df6_2.conda
+https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.4-nompi_py310hba70d50_102.conda
 https://conda.anaconda.org/conda-forge/noarch/oauthlib-3.2.2-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.0.0-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/tifffile-2022.4.26-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/xarray-2023.2.0-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.7.9-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/esmf-8.4.0-nompi_hdb2cfa9_4.conda
-https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.2-py310hc1b7723_6.conda
-https://conda.anaconda.org/conda-forge/noarch/python-geotiff-0.2.7-pyh6eeeb26_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/qt-main-5.15.8-h5d23da1_6.conda
-https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/distributed-2023.1.1-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/esmpy-8.4.0-nompi_py310h4c636dd_2.conda
-https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.0-py310ha325b7b_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/pyqt-5.15.7-py310hab646b1_3.conda
-https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/dask-2023.1.1-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/google-auth-2.16.0-pyh1a96a4e_1.conda
-https://conda.anaconda.org/conda-forge/linux-64/matplotlib-3.6.3-py310hff52083_0.conda
-https://conda.anaconda.org/conda-forge/noarch/owslib-0.27.2-pyhd8ed1ab_1.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.14-heaa33ce_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/parallelio-2.6.2-mpi_mpich_h79a22d0_100.conda
+https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.2.0-pyhd8ed1ab_1.conda
+https://conda.anaconda.org/conda-forge/noarch/tifffile-2023.8.30-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/xarray-2023.8.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.10.57-h6f6b8fa_21.conda
+https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.8.4-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/esmf-8.4.2-mpi_mpich_h2a0de38_103.conda
+https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.1-py310h5c4b078_7.conda
+https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h90689f9_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.56.3-h98fae49_0.conda
+https://conda.anaconda.org/conda-forge/noarch/python-geotiff-0.2.10-pyh6eeeb26_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/qt-main-5.15.8-h7fe3ca9_15.conda
+https://conda.anaconda.org/conda-forge/noarch/rdflib-sqlalchemy-0.5.3-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.15-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/distributed-2023.9.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/esmpy-8.4.2-pyhc1e730c_4.conda
+https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.4-py310h111440e_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/graphviz-8.1.0-h28d9a01_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libarrow-13.0.0-hb9dc469_0_cpu.conda
+https://conda.anaconda.org/conda-forge/linux-64/pyqt-5.15.9-py310h04931ad_4.conda
+https://conda.anaconda.org/conda-forge/noarch/requests-2.31.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/google-auth-2.22.0-pyh1a96a4e_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/matplotlib-3.7.2-py310hff52083_0.conda
+https://conda.anaconda.org/conda-forge/noarch/owslib-0.29.2-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/pooch-1.7.0-pyha770c72_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/pyarrow-13.0.0-py310hf9e7431_0_cpu.conda
+https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py310hff52083_3.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/requests-oauthlib-1.3.1-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/sphinx-6.1.3-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/torchvision-0.14.1-cpu_py310hdfb2906_0.conda
-https://conda.anaconda.org/conda-forge/noarch/google-auth-oauthlib-0.4.6-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/torchvision-0.15.2-cpu_py310hb9e6163_1.conda
+https://conda.anaconda.org/conda-forge/noarch/dask-2023.9.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/google-auth-oauthlib-1.0.0-pyhd8ed1ab_1.conda
+https://conda.anaconda.org/conda-forge/noarch/prov-2.0.0-pyhd3deb0d_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.2-py310ha4c1d20_0.conda
+https://conda.anaconda.org/conda-forge/noarch/sparse-0.14.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/tensorboard-2.14.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/xesmf-0.8-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/pywps-4.5.2-pyh6c4a22f_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/scipy-1.10.0-py310h8deb116_2.conda
-https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.21.1-py310hcb7e713_0.conda
-https://conda.anaconda.org/conda-forge/noarch/sparse-0.13.0-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/tensorboard-2.11.2-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/xesmf-0.7.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-1.0.7-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-1.0.5-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.0.4-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-1.0.6-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/sphinx-7.2.5-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.9-pyhd8ed1ab_0.conda

From 89f6fa4c76233151123178c10affd8cdacae6ac3 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Thu, 7 Sep 2023 20:19:29 +0200
Subject: [PATCH 32/49] using crai models in metadata

---
 duck/clintai.py               | 11 ++++++++---
 duck/processes/wps_clintai.py |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/duck/clintai.py b/duck/clintai.py
index 338c5e3..db31717 100644
--- a/duck/clintai.py
+++ b/duck/clintai.py
@@ -4,13 +4,16 @@
 import craimodels
 from climatereconstructionai import evaluate
 import logging
+
 LOGGER = logging.getLogger("PYWPS")
 
 info_models = craimodels.info_models()
 dataset_names = list(info_models.keys())
 
 
-def write_clintai_cfg(base_dir, model_dir, eval_name, data_name, data_type, eval_parameters):
+def write_clintai_cfg(
+    base_dir, model_dir, eval_name, data_name, data_type, eval_parameters
+):
     cfg_templ = """
     --data-root-dir {{ base_dir }}
     --log-dir {{ base_dir }}
@@ -31,7 +34,8 @@ def write_clintai_cfg(base_dir, model_dir, eval_name, data_name, data_type, eval
         eval_name=eval_name,
         data_name=data_name,
         data_type=data_type,
-        eval_parameters=eval_parameters)
+        eval_parameters=eval_parameters,
+    )
     out = base_dir / "clintai.cfg"
     with open(out, "w") as fp:
         fp.write(cfg)
@@ -52,7 +56,8 @@ def run(dataset, dataset_name, variable_name, outdir, update_status):
         eval_name=dataset.stem,
         data_name=dataset.name,
         data_type=variable_name,
-        eval_parameters=info_models[dataset_name]["eval-parameters"])
+        eval_parameters=info_models[dataset_name]["eval_parameters"],
+    )
     # print(f"written cfg {cfg_file}")
     try:
         evaluate(arg_file=cfg_file.as_posix(), prog_func=update_status)
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 4b609c1..1529717 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -86,7 +86,7 @@ def __init__(self):
             metadata=[
                 Metadata(
                     title="CRAI Info Model",
-                    href='{"test":1}',
+                    href=craimodels.raw_text(),
                     role=INFO_ROLE,
                 ),
                 Metadata(

From e11991fec9227413f1cb2dcc20c8d7eda1f811b0 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Thu, 7 Sep 2023 20:32:13 +0200
Subject: [PATCH 33/49] fixed path to provenance db

---
 duck/db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/duck/db.py b/duck/db.py
index 61b9f4b..634fd13 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -2,7 +2,7 @@
 from rdflib.plugins.sparql import prepareQuery
 
 # Provide the path to the SQLite database in the local folder
-DB_URL = "sqlite:///provenance.sqlite"
+DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
 
 
 class GraphDB(object):

From 3ed4aaab56a1a5187c311c8176719a94d53f265d Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Thu, 7 Sep 2023 20:43:09 +0200
Subject: [PATCH 34/49] fix metadata

---
 duck/processes/wps_clintai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 1529717..4cac187 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -86,7 +86,7 @@ def __init__(self):
             metadata=[
                 Metadata(
                     title="CRAI Info Model",
-                    href=craimodels.raw_text(),
+                    href=f"{craimodels.raw_text()}",
                     role=INFO_ROLE,
                 ),
                 Metadata(

From f2b374103e8a2159e93272db18304d0cd8d17867 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Thu, 7 Sep 2023 20:58:52 +0200
Subject: [PATCH 35/49] update dashboard

---
 duck/processes/wps_dashboard.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/duck/processes/wps_dashboard.py b/duck/processes/wps_dashboard.py
index 10b5b08..289aedb 100644
--- a/duck/processes/wps_dashboard.py
+++ b/duck/processes/wps_dashboard.py
@@ -14,6 +14,7 @@ def __init__(self):
                 abstract="The time period for the report seperated by /"
                 "Example: 2023-09-01/2023-09-30",
                 data_type="string",
+                default="2023-09-01/2023-09-30"
                 min_occurs=0,
                 max_occurs=1,
             ),
@@ -47,8 +48,9 @@ def _handler(self, request, response):
         # Generate an HTML report from the DataFrame
         html_report = self.write_html(df, workdir)
 
+        print(f"report: {html_report}")
         response.outputs["report"].file = html_report
-        response.outputs["report"].output_format = Format("text/html")
+        # response.outputs["report"].output_format = Format("text/html")
 
         return response
 

From 17db9256ce386cab72c0dc78cfc3127dd9308831 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Thu, 7 Sep 2023 21:04:23 +0200
Subject: [PATCH 36/49] fixed param

---
 duck/processes/wps_dashboard.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/duck/processes/wps_dashboard.py b/duck/processes/wps_dashboard.py
index 289aedb..f05e7f7 100644
--- a/duck/processes/wps_dashboard.py
+++ b/duck/processes/wps_dashboard.py
@@ -14,7 +14,7 @@ def __init__(self):
                 abstract="The time period for the report seperated by /"
                 "Example: 2023-09-01/2023-09-30",
                 data_type="string",
-                default="2023-09-01/2023-09-30"
+                default="2023-09-01/2023-09-30",
                 min_occurs=0,
                 max_occurs=1,
             ),

From d618abfd9bf5ac3ff2cd48ed45075d69fb2271f8 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Thu, 7 Sep 2023 21:08:22 +0200
Subject: [PATCH 37/49] fixed dashboard

---
 duck/processes/wps_dashboard.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/duck/processes/wps_dashboard.py b/duck/processes/wps_dashboard.py
index f05e7f7..70bbed8 100644
--- a/duck/processes/wps_dashboard.py
+++ b/duck/processes/wps_dashboard.py
@@ -36,6 +36,8 @@ def __init__(self):
             abstract="Generate an HTML report from a provenance database.",
             inputs=inputs,
             outputs=outputs,
+            status_supported=True,
+            store_supported=True,
         )
 
     def _handler(self, request, response):

From 5ef9239edf0c165f84f6aa95387a0c423d19f8b6 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Sun, 10 Sep 2023 12:06:48 +0200
Subject: [PATCH 38/49] working on provenance db config

---
 duck/db.py       | 5 ++++-
 duck/default.cfg | 3 +++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/duck/db.py b/duck/db.py
index 634fd13..fb7ba2b 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -1,8 +1,11 @@
 from rdflib import Graph, URIRef
 from rdflib.plugins.sparql import prepareQuery
+# from pywps import configuration
 
 # Provide the path to the SQLite database in the local folder
-DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
+# DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
+DB_URL = "sqlite:////tmp/provenance.sqlite"
+# DB_URL = configuration.get_config_value('provenance', 'db_url')
 
 
 class GraphDB(object):
diff --git a/duck/default.cfg b/duck/default.cfg
index 1c6c142..8fcec40 100644
--- a/duck/default.cfg
+++ b/duck/default.cfg
@@ -19,3 +19,6 @@ cleantempdir = false
 level = INFO
 file = duck.log
 format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s
+
+[provenance]
+db_url = sqlite:////tmp/provenance.sqlite

From 82de25dbec02342ca15023965e4c2fa503ebd5df Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Sun, 10 Sep 2023 13:49:29 +0200
Subject: [PATCH 39/49] update info_models

---
 duck/processes/wps_clintai.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 4cac187..f4cfaee 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -3,6 +3,7 @@
 import os
 from datetime import datetime
 import json
+import yaml
 import xarray as xr
 
 from pywps import Process
@@ -30,6 +31,8 @@
 INFO_ROLE = "https://clint.dkrz.de/spec/crai/info"
 
 models_list = list(craimodels.info_models().keys())
+info_models = yaml.safe_load(craimodels.raw_text())
+info_models_string = json.dumps(info_models, separators=(',', ':'))
 
 
 class ClintAI(Process):
@@ -86,7 +89,7 @@ def __init__(self):
             metadata=[
                 Metadata(
                     title="CRAI Info Model",
-                    href=f"{craimodels.raw_text()}",
+                    href=f"{info_models_string}",
                     role=INFO_ROLE,
                 ),
                 Metadata(

From db340f7cf2187f9e8fe3a08d59c2a47c23450448 Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Sun, 10 Sep 2023 14:25:44 +0200
Subject: [PATCH 40/49] use dummy models

---
 duck/db.py                    |  4 ++--
 duck/processes/wps_clintai.py | 43 ++++++++++++++++++++++++++++++++++-
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/duck/db.py b/duck/db.py
index fb7ba2b..ad3ae5f 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -3,8 +3,8 @@
 # from pywps import configuration
 
 # Provide the path to the SQLite database in the local folder
-# DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
-DB_URL = "sqlite:////tmp/provenance.sqlite"
+DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
+# DB_URL = "sqlite:////tmp/provenance.sqlite"
 # DB_URL = configuration.get_config_value('provenance', 'db_url')
 
 
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index f4cfaee..639a560 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -32,7 +32,48 @@
 
 models_list = list(craimodels.info_models().keys())
 info_models = yaml.safe_load(craimodels.raw_text())
-info_models_string = json.dumps(info_models, separators=(',', ':'))
+
+dummy_models = {
+    "HadCRUT4": [
+        {
+            "name": "dataset_name",
+            "value": "HadCRUT4",
+            "title": "HadCRUT4 Training dataset: 82133 samples of near surface temperature anomalies from the 20th century reanalysis dataset (https://psl.noaa.gov/data/20thC_Rean/) spanning the period 1870-2009. \nMask dataset: masks of missing values extracted from the HadCRUT4 dataset for the period 1850-2014."
+            
+        },
+        {
+            "name": "file",
+            "value": "https://www.metoffice.gov.uk/hadobs/hadcrut4/data/current/gridded_fields/HadCRUT.4.6.0.0.median_netcdf.zip",
+            "title": "Enter a URL to your HadCRUT4 netcdf file."
+        },
+        {
+            "name": "variable_name",
+            "value": "temperature_anomaly",
+            "title": "temperature_anomaly"
+        }
+    ],
+    "HadCRUT5": [
+        {
+            "name": "dataset_name",
+            "value": "HadCRUT5",
+            "title": "HadCRUT5 Training dataset: 82133 samples of near surface temperature anomalies from the 20th century reanalysis dataset (https://psl.noaa.gov/data/20thC_Rean/) spanning the period 1870-2009. \nMask dataset: masks of missing values extracted from the HadCRUT4 dataset for the period 1850-2014."
+            
+        },
+        {
+            "name": "file",
+            "value": "https://www.metoffice.gov.uk/hadobs/hadcrut5/data/current/non-infilled/HadCRUT.5.0.1.0.anomalies.ensemble_mean.nc",
+            "title": "Enter a URL to your HadCRUT5 netcdf file."
+        },
+        {
+            "name": "variable_name",
+            "value": "tas_mean",
+            "title": "tas_mean"
+        }
+    ]
+}
+
+# info_models_string = json.dumps(info_models, separators=(',', ':'))
+info_models_string = json.dumps(dummy_models, separators=(',', ':'))
 
 
 class ClintAI(Process):

From ffd05722b77206bba77c27ea7c41bd41d510675e Mon Sep 17 00:00:00 2001
From: Carsten Ehbrecht <ehbrecht@dkrz.de>
Date: Sun, 10 Sep 2023 15:10:04 +0200
Subject: [PATCH 41/49] fix stats hadcrud4

---
 duck/processes/wps_clintai.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 639a560..92856a2 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -219,11 +219,13 @@ def _handler(self, request, response):
                 (workdir / "outputs").rglob("*_infilled.nc"), key=os.path.getmtime
             )
 
+            outfile_for_stats = infiles[0]
             with ZipFile(outfile, "w") as zip:
                 for infile in infiles:
                     zip.write(infile, arcname=infile.as_posix().split("/")[-1])
         else:
             outfile = workdir / "outputs" / str(datasets[0].stem + "_infilled.nc")
+            outfile_for_stats = outfile
 
         response.outputs["output"].file = outfile
         response.outputs["plot"].file = (
@@ -232,7 +234,7 @@ def _handler(self, request, response):
 
         # stats
         datastats = DataStats(self.workdir)
-        datastats.gen_data_stats(outfile.as_posix(), variable_name)
+        datastats.gen_data_stats(outfile_for_stats.as_posix(), variable_name)
         stats = datastats.info["Mstats"]
 
         # prov

From 8dc4f4e6939df8a6edfb3e0719b62c0144836aea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20Pl=C3=A9siat?= <plesiat@dkrz.de>
Date: Tue, 23 Jan 2024 16:46:48 +0100
Subject: [PATCH 42/49] Retrieve model info with metadata structure from
 craimodels

---
 duck/processes/wps_clintai.py | 46 ++---------------------------------
 1 file changed, 2 insertions(+), 44 deletions(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 92856a2..98e775b 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -31,50 +31,8 @@
 INFO_ROLE = "https://clint.dkrz.de/spec/crai/info"
 
 models_list = list(craimodels.info_models().keys())
-info_models = yaml.safe_load(craimodels.raw_text())
-
-dummy_models = {
-    "HadCRUT4": [
-        {
-            "name": "dataset_name",
-            "value": "HadCRUT4",
-            "title": "HadCRUT4 Training dataset: 82133 samples of near surface temperature anomalies from the 20th century reanalysis dataset (https://psl.noaa.gov/data/20thC_Rean/) spanning the period 1870-2009. \nMask dataset: masks of missing values extracted from the HadCRUT4 dataset for the period 1850-2014."
-            
-        },
-        {
-            "name": "file",
-            "value": "https://www.metoffice.gov.uk/hadobs/hadcrut4/data/current/gridded_fields/HadCRUT.4.6.0.0.median_netcdf.zip",
-            "title": "Enter a URL to your HadCRUT4 netcdf file."
-        },
-        {
-            "name": "variable_name",
-            "value": "temperature_anomaly",
-            "title": "temperature_anomaly"
-        }
-    ],
-    "HadCRUT5": [
-        {
-            "name": "dataset_name",
-            "value": "HadCRUT5",
-            "title": "HadCRUT5 Training dataset: 82133 samples of near surface temperature anomalies from the 20th century reanalysis dataset (https://psl.noaa.gov/data/20thC_Rean/) spanning the period 1870-2009. \nMask dataset: masks of missing values extracted from the HadCRUT4 dataset for the period 1850-2014."
-            
-        },
-        {
-            "name": "file",
-            "value": "https://www.metoffice.gov.uk/hadobs/hadcrut5/data/current/non-infilled/HadCRUT.5.0.1.0.anomalies.ensemble_mean.nc",
-            "title": "Enter a URL to your HadCRUT5 netcdf file."
-        },
-        {
-            "name": "variable_name",
-            "value": "tas_mean",
-            "title": "tas_mean"
-        }
-    ]
-}
-
-# info_models_string = json.dumps(info_models, separators=(',', ':'))
-info_models_string = json.dumps(dummy_models, separators=(',', ':'))
-
+info_models = yaml.safe_load(craimodels.metadata())
+info_models_string = json.dumps(info_models, separators=(',', ':'))
 
 class ClintAI(Process):
     def __init__(self):

From 76ca1dbd47eaa0b13c06cdcbbc0cb258abbc65e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20Pl=C3=A9siat?= <plesiat@dkrz.de>
Date: Tue, 23 Jan 2024 17:08:06 +0100
Subject: [PATCH 43/49] Fix issue loading info models

---
 duck/processes/wps_clintai.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 98e775b..9cb630e 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -31,8 +31,7 @@
 INFO_ROLE = "https://clint.dkrz.de/spec/crai/info"
 
 models_list = list(craimodels.info_models().keys())
-info_models = yaml.safe_load(craimodels.metadata())
-info_models_string = json.dumps(info_models, separators=(',', ':'))
+info_models_string = json.dumps(craimodels.metadata(), separators=(',', ':'))
 
 class ClintAI(Process):
     def __init__(self):

From c110d9a349da18a71e8fc4beed772f3dc599a516 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Wed, 10 Apr 2024 17:58:44 +0200
Subject: [PATCH 44/49] disabled dashboard process

---
 duck/processes/__init__.py |  5 +++--
 tests/test_wps_caps.py     | 13 ++++++-------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/duck/processes/__init__.py b/duck/processes/__init__.py
index 9bccdb2..69d029b 100644
--- a/duck/processes/__init__.py
+++ b/duck/processes/__init__.py
@@ -1,7 +1,8 @@
 from .wps_clintai import ClintAI
-from .wps_dashboard import Dashboard
+
+# from .wps_dashboard import Dashboard
 
 processes = [
     ClintAI(),
-    Dashboard(),
+    # Dashboard(),
 ]
diff --git a/tests/test_wps_caps.py b/tests/test_wps_caps.py
index 6d0f9ba..e32a7ed 100644
--- a/tests/test_wps_caps.py
+++ b/tests/test_wps_caps.py
@@ -6,12 +6,11 @@
 
 def test_wps_caps():
     client = client_for(Service(processes=processes))
-    resp = client.get(service='wps', request='getcapabilities', version='1.0.0')
-    names = resp.xpath_text('/wps:Capabilities'
-                            '/wps:ProcessOfferings'
-                            '/wps:Process'
-                            '/ows:Identifier')
+    resp = client.get(service="wps", request="getcapabilities", version="1.0.0")
+    names = resp.xpath_text(
+        "/wps:Capabilities" "/wps:ProcessOfferings" "/wps:Process" "/ows:Identifier"
+    )
     assert sorted(names.split()) == [
-        'crai',
-        'dashboard',
+        "crai",
+        # 'dashboard',
     ]

From bef0b9c6a0d2299d29f9605f4430d98cea73c629 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Fri, 12 Apr 2024 15:23:37 +0200
Subject: [PATCH 45/49] using configured db

---
 duck/db.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/duck/db.py b/duck/db.py
index ad3ae5f..ecb5408 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -1,11 +1,13 @@
 from rdflib import Graph, URIRef
 from rdflib.plugins.sparql import prepareQuery
-# from pywps import configuration
+
+from pywps import configuration
 
 # Provide the path to the SQLite database in the local folder
-DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
+# DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
 # DB_URL = "sqlite:////tmp/provenance.sqlite"
 # DB_URL = configuration.get_config_value('provenance', 'db_url')
+DB_URL = configuration.get_config_value("logging", "database")
 
 
 class GraphDB(object):

From 7fe5bec32a6ba7f112f30cf5470ae79aecd4868f Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Fri, 12 Apr 2024 16:00:25 +0200
Subject: [PATCH 46/49] update db

---
 duck/db.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/duck/db.py b/duck/db.py
index ecb5408..0c3b3fe 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -6,8 +6,7 @@
 # Provide the path to the SQLite database in the local folder
 # DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
 # DB_URL = "sqlite:////tmp/provenance.sqlite"
-# DB_URL = configuration.get_config_value('provenance', 'db_url')
-DB_URL = configuration.get_config_value("logging", "database")
+DB_URL = configuration.get_config_value("provenance", "db_uri")
 
 
 class GraphDB(object):

From c6f095db128f88f2c1e26d3a4bcfd26be91ae82a Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Fri, 12 Apr 2024 16:26:34 +0200
Subject: [PATCH 47/49] revert db

---
 duck/db.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/duck/db.py b/duck/db.py
index 0c3b3fe..4c6fc17 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -1,12 +1,12 @@
 from rdflib import Graph, URIRef
 from rdflib.plugins.sparql import prepareQuery
 
-from pywps import configuration
+# from pywps import configuration
 
 # Provide the path to the SQLite database in the local folder
-# DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
+DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
 # DB_URL = "sqlite:////tmp/provenance.sqlite"
-DB_URL = configuration.get_config_value("provenance", "db_uri")
+# DB_URL = configuration.get_config_value("provenance", "db_uri")
 
 
 class GraphDB(object):

From 4e99064b340e32a652395abbbcb5cba35ffeb18f Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Fri, 12 Apr 2024 16:46:27 +0200
Subject: [PATCH 48/49] working on prov store

---
 duck/db.py                    | 6 +++---
 duck/processes/wps_clintai.py | 8 ++++++--
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/duck/db.py b/duck/db.py
index 4c6fc17..1f8525d 100644
--- a/duck/db.py
+++ b/duck/db.py
@@ -1,12 +1,12 @@
 from rdflib import Graph, URIRef
 from rdflib.plugins.sparql import prepareQuery
 
-# from pywps import configuration
+from pywps import configuration
 
 # Provide the path to the SQLite database in the local folder
-DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
+# DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
 # DB_URL = "sqlite:////tmp/provenance.sqlite"
-# DB_URL = configuration.get_config_value("provenance", "db_uri")
+DB_URL = configuration.get_config_value("logging", "database")
 
 
 class GraphDB(object):
diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 9cb630e..1acd92d 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -31,7 +31,8 @@
 INFO_ROLE = "https://clint.dkrz.de/spec/crai/info"
 
 models_list = list(craimodels.info_models().keys())
-info_models_string = json.dumps(craimodels.metadata(), separators=(',', ':'))
+info_models_string = json.dumps(craimodels.metadata(), separators=(",", ":"))
+
 
 class ClintAI(Process):
     def __init__(self):
@@ -214,7 +215,10 @@ def _handler(self, request, response):
             start_time,
             end_time,
         )
-        prov.store_rdf()
+        try:
+            prov.store_rdf()
+        except Exception:
+            LOGGER.exception("failed to store provenance record!")
         # prov end
 
         response.update_status("done.", 100)

From a0f4ea3f3fdccf6125b7e59406c974baf0c108d3 Mon Sep 17 00:00:00 2001
From: Pingu Carsti <ehbrecht@dkrz.de>
Date: Fri, 12 Apr 2024 17:09:47 +0200
Subject: [PATCH 49/49] skip histogram in provenance

---
 duck/processes/wps_clintai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/duck/processes/wps_clintai.py b/duck/processes/wps_clintai.py
index 1acd92d..4da5280 100644
--- a/duck/processes/wps_clintai.py
+++ b/duck/processes/wps_clintai.py
@@ -208,7 +208,7 @@ def _handler(self, request, response):
                 "mean": stats["mean"],
                 "stddev": stats["std"],
                 "info": json.dumps(datastats.info, separators=(",", ":")),
-                "histogram": datastats.histogram,
+                # "histogram": datastats.histogram,
             },
             [datasets[0].as_posix()],
             [f"{datasets[0].as_posix()}_infilled.nc"],