From f0a4e70693329def1f2f86a5f157008e92502d07 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 29 Apr 2026 12:15:58 -1000 Subject: [PATCH] Documentation++ --- docs/source/analyzing_instances.rst | 65 +++++++++++++++---- .../source/generating_workflow_benchmarks.rst | 17 +++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/docs/source/analyzing_instances.rst b/docs/source/analyzing_instances.rst index 64ec4179..13e11071 100644 --- a/docs/source/analyzing_instances.rst +++ b/docs/source/analyzing_instances.rst @@ -68,28 +68,34 @@ method. Supported log parsers +++++++++++++++++++++ -- :class:`~wfcommons.wfinstances.logs.pegasusrec.HierarchicalPegasusLogsParser` - :class:`~wfcommons.wfinstances.logs.makeflow.MakeflowLogsParser` - :class:`~wfcommons.wfinstances.logs.nextflow.NextflowLogsParser` - :class:`~wfcommons.wfinstances.logs.pegasus.PegasusLogsParser` +- :class:`~wfcommons.wfinstances.logs.snakemake.SnakemakeLogsParser` +- :class:`~wfcommons.wfinstances.logs.ro_create.ROCrateLogsParser` - :class:`~wfcommons.wfinstances.logs.taskvine.TaskVineLogsParser` Examples ++++++++ -Hierarchical Pegasus -++++++++++++++++++++ +Below we give basic examples for using the log parsers. Review the specific documentation +of each log parser for more details, such as additional parameters that configure the behavior +of the log parser. -This parser targets Pegasus submit directories that contain hierarchical workflows. -It recursively parses sub-workflows and rebuilds a coherent workflow instance:: +.. + Hierarchical Pegasus + ++++++++++++++++++++ - import pathlib - from wfcommons.wfinstances import HierarchicalPegasusLogsParser + This parser targets Pegasus submit directories that contain hierarchical workflows. + It recursively parses sub-workflows and rebuilds a coherent workflow instance:: + + import pathlib + from wfcommons.wfinstances import HierarchicalPegasusLogsParser - submit_dir = pathlib.Path('/path/to/pegasus/hierarchical/submit/dir/') - parser = HierarchicalPegasusLogsParser(submit_dir=submit_dir) - workflow = parser.build_workflow('pegasus-hierarchical-workflow-test') - workflow.write_json(pathlib.Path('./pegasus-hierarchical-workflow.json')) + submit_dir = pathlib.Path('/path/to/pegasus/hierarchical/submit/dir/') + parser = HierarchicalPegasusLogsParser(submit_dir=submit_dir) + workflow = parser.build_workflow('pegasus-hierarchical-workflow-test') + workflow.write_json(pathlib.Path('./pegasus-hierarchical-workflow.json')) Makeflow ++++++++ @@ -192,6 +198,43 @@ class: :: workflow_path = pathlib.Path('./pegasus-workflow.json') workflow.write_json(workflow_path) + + +RO-Crate ++++++++++ +`RO-Crate `_ is a format for packaging research data so as to promote +open and reproducible science. The RO-Crate logs parses processes RO-Crate artifacts created by executing workflows +using the `Streamflow `_ workflow management system. It may work +for RO-Crate generated using other systems, but has not been tested. +It takes as input the path of the RO-Crate directory, which contains the `ro-crate-metadata.json` file. It +generates workflow instances compatible with :ref:`json-format-label`:: + + import pathlib + from wfcommons.wfinstances import ROCrateLogsParser + + execution_dir = pathlib.Path('/path/to/snakemake/execution/dir/') + parser = ROCrateLogsParser(execution_dir=execution_dir) + workflow = parser.build_workflow('ro-crate-workflow-test') + workflow.write_json(pathlib.Path('./ro-crate-workflow.json')) + +Snakemake ++++++++++ +`Snakemake `_ is a popular and easy-to-use workflow system. +The Snakemake logs parser processes execution logs generated by the Snakemake +`snkmt plugin `_. +It takes as input the path of the directory where all workflow data files reside (input and output +of workflow tasks), and the path of the sqlite database created by the snkmt plugin. It +generates workflow instances compatible with :ref:`json-format-label`:: + + import pathlib + from wfcommons.wfinstances import SnakemakeLogsParser + + execution_dir = pathlib.Path('/path/to/snakemake/execution/dir/') + snkmt_db = pathlib.Path('/path/to/snkmt/sqlite/databasefile/') + parser = SnakemakeLogsParser(execution_dir=execution_dir, snkmt_db=snkmt_db) + workflow = parser.build_workflow('snakemake-workflow-test') + workflow.write_json(pathlib.Path('./snakemake-workflow.json')) + TaskVine ++++++++ diff --git a/docs/source/generating_workflow_benchmarks.rst b/docs/source/generating_workflow_benchmarks.rst index a980e2e0..062e2e81 100644 --- a/docs/source/generating_workflow_benchmarks.rst +++ b/docs/source/generating_workflow_benchmarks.rst @@ -89,6 +89,7 @@ Supported translators (alphabetical) - Parsl - Pegasus - PyCOMPSs +- Snakemake - Swift/T - TaskVine @@ -283,6 +284,22 @@ parallel Python applications on distributed infrastructures:: translator = PyCompssTranslator(benchmark.workflow) translator.translate(output_folder=pathlib.Path("./pycompss-wf/")) +Snakemake ++++++++++ + +`Snakemake `_ is a workflow system to create data +analysis workflows using a human readable, Python-based language:: + + import pathlib + from wfcommons import BlastRecipe + from wfcommons.wfbench import WorkflowBenchmark, SnakemakeTranslator + + benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=500) + benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=100, data=10, percent_cpu=1.0) + + translator = SnakemakeTranslator(benchmark.workflow) + translator.translate(output_folder=pathlib.Path("./snakemake/")) + Swift/T +++++++