Complete invoice PDF workflow with durable providers, real browser and legacy compatibility

.github/workflows/ci-cd.yml

@@ -1,6 +1,7 @@
 name: InvoiceFlow CI/CD Pipeline
 
 on:
+  workflow_dispatch:
   push:
     branches: [ main, develop ]
   pull_request:
@@ -17,22 +18,24 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v3
         with:
-          terraform_version: 1.6.0
+          terraform_version: 1.15.9
 
       - name: Terraform Format Check
         run: |
-          find infra/ -name "*.tf" -exec terraform fmt -check=true {} \;
+          terraform fmt -check -recursive infra/
 
       - name: Terraform Validation
         run: |
           for env in dev staging prod; do
             echo "Validating $env environment..."
             cd infra/environments/$env
-            terraform init -backend=false
+            terraform init -backend=false -input=false
             terraform validate
             cd ../../..
           done
@@ -47,11 +50,18 @@ jobs:
         with:
           directory: infra/
           framework: terraform
+          skip_download: true
+          skip_results_upload: true
+          download_external_modules: false
 
   # Security scanning
   security-scan:
     name: Security Scan
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+      actions: read
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -65,7 +75,7 @@ jobs:
           output: 'trivy-results.sarif'
 
       - name: Upload Trivy scan results
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: 'trivy-results.sarif'
 
@@ -76,87 +86,27 @@ jobs:
     needs: [auth-service-tests, match-service-tests]
     timeout-minutes: 20
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
         with:
-          python-version: '3.11'
-
-      - name: Install e2e test dependencies
-        run: |
-          cd services/e2e
-          pip install -r requirements.txt
-
-      - name: Build service images
-        run: |
-          # Build images for all services
-          if [ -d "services/ingest" ]; then
-            docker build -t invoiceflow/ingest:e2e services/ingest/
-          fi
-          if [ -d "services/extract" ]; then
-            docker build -t invoiceflow/extract:e2e services/extract/
-          fi
-          if [ -d "services/match" ]; then
-            docker build -t invoiceflow/match:e2e services/match/
-          fi
-
-      - name: Start services with docker-compose
-        run: |
-          docker-compose -f docker-compose.e2e.yml up -d
-
-          # Wait for services to be healthy
-          timeout=120
-          elapsed=0
-
-          while [ $elapsed -lt $timeout ]; do
-            if docker-compose -f docker-compose.e2e.yml ps | grep -q "healthy"; then
-              ingest_health=$(docker inspect invoiceflow-ingest-e2e --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown")
-              extract_health=$(docker inspect invoiceflow-extract-e2e --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown")
-              match_health=$(docker inspect invoiceflow-match-e2e --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown")
-
-              if [ "$ingest_health" = "healthy" ] && [ "$extract_health" = "healthy" ] && [ "$match_health" = "healthy" ]; then
-                echo "All services are healthy"
-                break
-              fi
-            fi
-
-            echo "Waiting for services to become healthy... ($elapsed/$timeout)"
-            sleep 10
-            elapsed=$((elapsed + 10))
-          done
-
-          if [ $elapsed -ge $timeout ]; then
-            echo "Services failed to become healthy"
-            docker-compose -f docker-compose.e2e.yml ps
-            docker-compose -f docker-compose.e2e.yml logs
-            exit 1
-          fi
-
-      - name: Run end-to-end tests
-        timeout-minutes: 5
-        run: |
-          cd services/e2e
-          python -m pytest test_full_pipeline.py -v --timeout=300
-
-      - name: Upload test results
-        uses: actions/upload-artifact@v3
-        if: always()
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - uses: actions/setup-python@v5
         with:
-          name: e2e-test-results
-          path: |
-            services/e2e/htmlcov/
-            services/e2e/coverage.xml
-
-      - name: Cleanup services
-        if: always()
-        run: |
-          docker-compose -f docker-compose.e2e.yml down --remove-orphans
-          docker-compose -f docker-compose.e2e.yml rm -f
+          python-version: '3.11'
+      - run: pip install -r services/pipeline/requirements.txt
+      - run: python -m pytest tests/pipeline tests/legacy/test_document_contract.py -q
+      - run: docker compose -f docker-compose.e2e.yml up -d postgres rabbitmq localstack --wait --wait-timeout 180
+      - run: python scripts/check_legacy_storage.py --bootstrap
+      - run: docker compose -f docker-compose.e2e.yml up -d --build --wait --wait-timeout 240
+      - run: python -m pytest services/e2e/test_full_pipeline.py -v
+      - if: always()
+        run: docker compose -f docker-compose.e2e.yml logs --no-color | tee legacy-provider-e2e.log
+      - if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: legacy-provider-e2e
+          path: legacy-provider-e2e.log
+      - if: always()
+        run: docker compose -f docker-compose.e2e.yml down --remove-orphans
 
   # Auth Service Testing
   auth-service-tests:
@@ -199,6 +149,7 @@ jobs:
           cd services/auth
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          pip install flake8 mypy pytest-cov
 
       - name: Lint Python code
         run: |
@@ -220,6 +171,15 @@ jobs:
           cd services/auth
           python -m pytest tests/ -v --cov=app --cov-report=xml --cov-report=term-missing
 
+      - name: Real PostgreSQL auth HTTP roundtrip
+        env:
+          DATABASE_URL: postgresql://testuser:testpassword@localhost:5432/testdb
+          JWT_SECRET: test-secret-key-that-is-long-enough-for-testing
+          ENVIRONMENT: testing
+        run: |
+          cd services/auth
+          python scripts/check_native_auth.py
+
       - name: Upload auth service coverage
         uses: codecov/codecov-action@v3
         with:
@@ -277,6 +237,7 @@ jobs:
         run: |
           cd services/match
           pip install -r requirements.txt
+          pip install flake8 black isort "mypy==1.11.2"
 
       - name: Lint match service code
         run: |
@@ -355,175 +316,10 @@ jobs:
   # E2E Testing
   e2e-tests:
     name: End-to-End Tests
-    runs-on: ubuntu-latest
     needs: [auth-service-tests, frontend-tests]
-    services:
-      postgres:
-        image: postgres:15
-        env:
-          POSTGRES_PASSWORD: testpassword
-          POSTGRES_USER: testuser
-          POSTGRES_DB: testdb
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: '18'
-          cache: 'npm'
-          cache-dependency-path: frontend/package-lock.json
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-
-      - name: Install Python dependencies
-        run: |
-          cd services/auth
-          pip install -r requirements.txt
-
-      - name: Install frontend dependencies
-        run: |
-          cd frontend
-          npm ci
-
-      - name: Install Playwright
-        run: |
-          cd frontend
-          npx playwright install --with-deps
-
-      - name: Start auth service
-        env:
-          DATABASE_URL: postgresql://testuser:testpassword@localhost:5432/testdb
-          JWT_SECRET: test-secret-key-that-is-long-enough-for-testing
-          ENVIRONMENT: testing
-        run: |
-          cd services/auth
-          python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 &
-          sleep 10
-
-      - name: Start frontend
-        env:
-          VITE_AUTH_SERVICE_URL: http://localhost:8000
-        run: |
-          cd frontend
-          npm run build
-          npm run preview --port 3000 &
-          sleep 10
-
-      - name: Run E2E tests
-        run: |
-          cd frontend
-          npm run test:e2e
-
-      - name: Upload E2E test results
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: frontend/playwright-report/
+    uses: ./.github/workflows/durable-pipeline.yml
 
   # End-to-End Pipeline Tests
-  e2e-pipeline:
-    name: End-to-End Pipeline Tests
-    runs-on: ubuntu-latest
-    needs: [auth-service-tests, match-service-tests]
-    timeout-minutes: 20
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-
-      - name: Install e2e test dependencies
-        run: |
-          cd services/e2e
-          pip install -r requirements.txt
-
-      - name: Build service images
-        run: |
-          # Build images for all services
-          if [ -d "services/ingest" ]; then
-            docker build -t invoiceflow/ingest:e2e services/ingest/
-          fi
-          if [ -d "services/extract" ]; then
-            docker build -t invoiceflow/extract:e2e services/extract/
-          fi
-          if [ -d "services/match" ]; then
-            docker build -t invoiceflow/match:e2e services/match/
-          fi
-
-      - name: Start services with docker-compose
-        run: |
-          docker-compose -f docker-compose.e2e.yml up -d
-
-          # Wait for services to be healthy
-          timeout=120
-          elapsed=0
-
-          while [ $elapsed -lt $timeout ]; do
-            if docker-compose -f docker-compose.e2e.yml ps | grep -q "healthy"; then
-              ingest_health=$(docker inspect invoiceflow-ingest-e2e --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown")
-              extract_health=$(docker inspect invoiceflow-extract-e2e --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown")
-              match_health=$(docker inspect invoiceflow-match-e2e --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown")
-
-              if [ "$ingest_health" = "healthy" ] && [ "$extract_health" = "healthy" ] && [ "$match_health" = "healthy" ]; then
-                echo "All services are healthy"
-                break
-              fi
-            fi
-
-            echo "Waiting for services to become healthy... ($elapsed/$timeout)"
-            sleep 10
-            elapsed=$((elapsed + 10))
-          done
-
-          if [ $elapsed -ge $timeout ]; then
-            echo "Services failed to become healthy"
-            docker-compose -f docker-compose.e2e.yml ps
-            docker-compose -f docker-compose.e2e.yml logs
-            exit 1
-          fi
-
-      - name: Run end-to-end tests
-        timeout-minutes: 5
-        run: |
-          cd services/e2e
-          python -m pytest test_full_pipeline.py -v --timeout=300
-
-      - name: Upload test results
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: e2e-test-results
-          path: |
-            services/e2e/htmlcov/
-            services/e2e/coverage.xml
-
-      - name: Cleanup services
-        if: always()
-        run: |
-          docker-compose -f docker-compose.e2e.yml down --remove-orphans
-          docker-compose -f docker-compose.e2e.yml rm -f
-
-  # Build and push Docker images
   build-images:
     name: Build Docker Images
     runs-on: ubuntu-latest