orchestrator/examples/simple_data_processing.yaml at main · ContextLab/orchestrator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
# Simple Data Processing Pipeline
# Uses real tools: filesystem and data-processing
id: simple_data_processing
name: Simple Data Processing Pipeline
description: Read a CSV file, process it, and save results
version: "1.0.0"

parameters:
  output_path:
    type: string
    default: "examples/outputs/simple_data_processing"
    description: Directory where output files will be saved

steps:
  - id: read_data
    tool: filesystem
    action: read
    parameters:
      path: "data/input.csv"

  - id: process_data
    tool: data-processing
    action: filter
    parameters:
      data: "{{ read_data.result.content }}"
      format: "csv"
      operation:
        criteria:
          status: "active"
    dependencies:
      - read_data

  - id: save_results
    tool: filesystem
    action: write
    parameters:
      path: "{{ output_path }}/output_{{ execution.timestamp | slugify }}.csv"
      content: "{{ process_data.processed_data }}"
    dependencies:
      - process_data

  - id: save_report
    tool: filesystem
    action: write
    parameters:
      path: "{{ output_path }}/report_{{ execution.timestamp | slugify }}.md"
      content: |
        # Simple Data Processing Report

        **Generated:** {{ execution.timestamp }}
        **Pipeline:** Simple Data Processing

        ## Processing Summary

        - **Input File:** data/input.csv
        - **Filter Applied:** status = "active"
        - **Output File:** {{ output_path }}/output_{{ execution.timestamp | slugify }}.csv

        ## Results

        The data processing pipeline successfully filtered the input CSV file to include only records with status="active".

        ### Filtered Data Preview:
        ```csv
        {{ process_data.processed_data | truncate(500) }}
        ```

        ---
        *Generated by Simple Data Processing Pipeline*
    dependencies:
      - process_data