Curiousduck

.gitignore

@@ -111,3 +111,6 @@ testdata.json
 
 # macos
 .DS_Store
+
+# tests
+PET0.ESMF_LogFile

duck/clintai.py

@@ -4,13 +4,16 @@
 import craimodels
 from climatereconstructionai import evaluate
 import logging
+
 LOGGER = logging.getLogger("PYWPS")
 
 info_models = craimodels.info_models()
 dataset_names = list(info_models.keys())
 
 
-def write_clintai_cfg(base_dir, model_dir, eval_name, data_name, data_type, eval_parameters):
+def write_clintai_cfg(
+    base_dir, model_dir, eval_name, data_name, data_type, eval_parameters
+):
     cfg_templ = """
     --data-root-dir {{ base_dir }}
     --log-dir {{ base_dir }}
@@ -31,7 +34,8 @@ def write_clintai_cfg(base_dir, model_dir, eval_name, data_name, data_type, eval
         eval_name=eval_name,
         data_name=data_name,
         data_type=data_type,
-        eval_parameters=eval_parameters)
+        eval_parameters=eval_parameters,
+    )
     out = base_dir / "clintai.cfg"
     with open(out, "w") as fp:
         fp.write(cfg)
@@ -52,7 +56,8 @@ def run(dataset, dataset_name, variable_name, outdir, update_status):
         eval_name=dataset.stem,
         data_name=dataset.name,
         data_type=variable_name,
-        eval_parameters=info_models[dataset_name]["eval-parameters"])
+        eval_parameters=info_models[dataset_name]["eval_parameters"],
+    )
     # print(f"written cfg {cfg_file}")
     try:
         evaluate(arg_file=cfg_file.as_posix(), prog_func=update_status)

duck/data_stats.py

@@ -0,0 +1,96 @@
+import xarray as xr
+import matplotlib.pyplot as plt
+import numpy as np
+import yaml
+import pathlib
+import io
+import base64
+
+
+def get_stats(data):
+    return {
+        "min": float(np.nanmin(data)),
+        "max": float(np.nanmax(data)),
+        "mean": float(np.nanmean(data)),
+        "std": float(np.nanstd(data)),
+    }
+
+
+class DataStats(object):
+    def __init__(self, output_dir):
+        if isinstance(output_dir, pathlib.Path):
+            self.output_dir = output_dir
+        else:
+            self.output_dir = pathlib.Path(output_dir)
+        self.info = None
+        self.histogram = None
+
+    def gen_data_stats(self, filename, var, nbins=100):
+        ds = xr.open_dataset(filename)
+
+        vstats = get_stats(ds[var].values)
+        bins = np.linspace(vstats["min"], vstats["max"], num=nbins + 1)
+
+        ntime, nlon, nlat = ds[var].shape
+        mratio = np.zeros(ntime)
+        hist = np.zeros((ntime, nbins))
+        for i in range(ntime):
+            a = ds[var].values[i]
+            idx = ~np.isnan(a)
+            mratio[i] = idx.sum()
+            a = np.histogram(a[idx], bins=bins)[0]
+            hist[i] = a / max(a)
+
+        mratio = 1 - mratio / (nlon * nlat)
+
+        # TODO: It would be great to store the distribution graph in a database
+        if True:
+            plt.close()
+            plt.imshow(
+                hist,
+                aspect="auto",
+                origin="lower",
+                extent=[vstats["min"], vstats["max"], 0, ntime],
+                cmap="gist_ncar",
+            )
+            ax = plt.gca()
+            ax.grid(color="gray", linestyle="-.", linewidth=1)
+            plt.xlabel(var)
+            plt.ylabel("Timesteps")
+            # outfile = self.output_dir / "histogram.png"
+            # print(f"histogram: {outfile}")
+            # plt.savefig(outfile.as_posix(), dpi=50)
+            # store as base64
+            # Save the plot to a BytesIO object
+            buffer = io.BytesIO()
+            plt.savefig(buffer, format="png")
+            buffer.seek(0)
+
+            # Encode the BytesIO object as base64
+            base64_encoded_plot = base64.b64encode(buffer.read()).decode("utf-8")
+            # print(f"{base64_encoded_plot}")
+            self.histogram = base64_encoded_plot
+            # close plot
+            plt.close()
+
+        # The following information should be stored in a database
+        attrs = {}
+        orig_attrs = dict(ds.attrs)
+        for key in orig_attrs:
+            value = orig_attrs[key]
+            if isinstance(value, str):
+                attrs[key] = value
+
+        self.info = {}
+        self.info["Attrs"] = attrs
+        self.info["Dims"] = dict(ds.dims)
+        self.info["Vars"] = list(dict(ds.variables).keys())
+        self.info["Vstats"] = vstats
+        self.info["Mstats"] = get_stats(mratio)
+        # print(self.info)
+
+    def write_json(self):
+        outfile = self.output_dir / "info.txt"
+        with open(outfile.as_posix(), "w") as f:
+            yaml.dump(self.info, f)
+        return outfile

duck/db.py

@@ -0,0 +1,42 @@
+from rdflib import Graph, URIRef
+from rdflib.plugins.sparql import prepareQuery
+
+from pywps import configuration
+
+# Provide the path to the SQLite database in the local folder
+# DB_URL = "sqlite:////var/lib/pywps/db/provenance.sqlite"
+# DB_URL = "sqlite:////tmp/provenance.sqlite"
+DB_URL = configuration.get_config_value("logging", "database")
+
+
+class GraphDB(object):
+    def __init__(self):
+        # Create a graph with a specific backend store
+        self.graph = Graph(
+            store="SQLAlchemy", identifier=URIRef("http://example.org/graph")
+        )
+        self.graph.open(DB_URL, create=True)
+
+    def add(self, data):
+        new_graph = Graph()
+        new_graph.parse(data=data, format="turtle")
+
+        # add rdf to existing graph
+        for triple in new_graph:
+            self.graph.add(triple)
+        # Commit changes to the store
+        self.graph.commit()
+
+    def query(self, query_str):
+        namespaces = {
+            "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+            "foaf": "http://xmlns.com/foaf/0.1/",
+            "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
+            "prov": "http://www.w3.org/ns/prov#",
+            "provone": "http://purl.dataone.org/provone/2015/01/15/ontology#",
+            "dcterms": "http://purl.org/dc/terms/",
+            "clint": "urn:clint:",
+        }
+        query = prepareQuery(query_str, initNs=namespaces)
+        results = self.graph.query(query)
+        return results

duck/default.cfg

@@ -19,3 +19,6 @@ cleantempdir = false
 level = INFO
 file = duck.log
 format = %(asctime)s] [%(levelname)s] line=%(lineno)s module=%(module)s %(message)s
+
+[provenance]
+db_url = sqlite:////tmp/provenance.sqlite

duck/processes/__init__.py

@@ -1,5 +1,8 @@
 from .wps_clintai import ClintAI
 
+# from .wps_dashboard import Dashboard
+
 processes = [
     ClintAI(),
+    # Dashboard(),
 ]