data_processing.yaml

id: data-processing
name: Data Processing Pipeline
description: Process and validate data from various sources

parameters:
  data_source:
    type: string
    required: false
    default: "examples/test_data/sample_data.json"
    description: Path to data file (CSV or JSON)
  output_format:
    type: string
    default: json
    description: Output format (json, csv, or yaml)
  output_path:
    type: string
    default: "examples/outputs/data_processing"
    description: Directory where output files will be saved

steps:
  - id: load_data
    tool: filesystem
    action: read
    parameters:
      path: "{{ data_source }}"
  
  - id: parse_data
    action: generate_text
    parameters:
      prompt: |
        Parse this data and identify its structure:
        {{ load_data }}
        
        Return ONLY one word: "json" if it's JSON, "csv" if it's CSV, or "unknown" if unclear.
      model: <AUTO task="parse">Select a model for parsing</AUTO>
      max_tokens: 10
    dependencies:
      - load_data
  
  - id: validate_data
    tool: validation
    action: validate
    parameters:
      data: "{{ load_data.result.content }}"
      schema:
        type: object
        properties:
          records:
            type: array
            items:
              type: object
              properties:
                id:
                  type: integer
                name:
                  type: string
                active:
                  type: boolean
              required: ["id", "name"]
      validation_level: permissive
    dependencies:
      - parse_data
  
  - id: transform_data
    tool: data-processing
    action: transform
    parameters:
      data: "{{ load_data.result.content }}"
      operation:
        transformations:
          - type: filter
            field: active
            value: true
          - type: aggregate
            operation: sum
            field: value
    dependencies:
      - validate_data
  
  - id: format_results
    action: generate_text
    parameters:
      prompt: |
        Convert this data to clean JSON format:
        {{ transform_data }}
        
        Return ONLY valid JSON without any markdown formatting, code fences, or explanations.
        Do NOT include ```json or ``` markers.
        Start directly with { and end with }
      model: <AUTO task="format">Select a model for formatting</AUTO>
      max_tokens: 500
    dependencies:
      - transform_data
  
  - id: save_results
    tool: filesystem
    action: write
    parameters:
      path: "{{ output_path }}/processed_data.{{ output_format }}"
      content: "{{ format_results }}"
    dependencies:
      - format_results
      
  - id: generate_summary
    action: generate_text
    parameters:
      prompt: |
        Generate a brief processing summary based on:
        - Original data: {{ load_data }}
        - Validation result: {{ validate_data }}
        - Transformed data: {{ transform_data }}
        
        Include:
        - Number of records processed
        - Validation status
        - Transformation applied
        
        Keep it concise (3-4 lines).
      model: <AUTO task="summary">Select a model for summary</AUTO>
      max_tokens: 150
    dependencies:
      - transform_data
      
  - id: save_report
    tool: filesystem
    action: write
    parameters:
      path: "{{ output_path }}/processing_report.md"
      content: |
        # Data Processing Report
        
        **Source File:** {{ data_source }}
        **Output Format:** {{ output_format }}
        
        ## Validation Results
        
        - Validation Status: {% if validate_data.valid %}Passed{% else %}Failed{% endif %}
        - Errors: {% if validate_data.errors %}{{ validate_data.errors | length }} errors found{% else %}None{% endif %}
        - Warnings: {% if validate_data.warnings %}{{ validate_data.warnings | length }} warnings{% else %}None{% endif %}
        
        ## Processing Summary
        
        {{ generate_summary }}
        
        ## Output Details
        
        - Transformed data saved to: {{ output_path }}/processed_data.{{ output_format }}
        - Report generated at: {{ output_path }}/processing_report.md
        
        ---
        *Generated by Data Processing Pipeline*
    dependencies:
      - save_results
      - generate_summary

outputs:
  original_data: "{{ load_data }}"
  validated: "{{ validate_data.valid }}"
  transformed: "{{ transform_data }}"
  output_file: "{{ output_path }}/processed_data.{{ output_format }}"
  summary: "{{ generate_summary }}"