Initial setup of the repository

.github/workflows/pr_validation.yml

@@ -0,0 +1,59 @@
+name: Pull Request Data Validation
+
+on:
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  validate_sql:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Dependencies
+        run: |
+          pip install dbt-bigquery sqlfluff-templater-dbt
+
+      - name: Authenticate to GCP Natively
+        run: |
+          # Write the secret JSON token safely as a raw literal string
+          cat << 'EOF' > ${HOME}/gcp_key.json
+          ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+          EOF
+
+          # Activate the service account using the pre-installed gcloud CLI
+          gcloud auth activate-service-account --key-file=${HOME}/gcp_key.json
+
+          # Set the application default credentials environment variable for dbt
+          echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/gcp_key.json" >> $GITHUB_ENV
+
+      # Generate a local connection profile using service-account credentials
+      - name: Create dynamic profiles.yml for dbt
+        run: |
+          cat << EOF > profiles.yml
+          oppia_analytics:
+            target: dev
+            outputs:
+              dev:
+                type: bigquery
+                method: service-account
+                keyfile: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
+                project: oppia-analytics-test
+                dataset: dev_github_actions
+                threads: 4
+                timeout_seconds: 300
+                location: US
+          EOF
+
+      - name: Install dbt Packages (if any)
+        run: dbt deps --profiles-dir .
+
+      - name: Build and Test Models
+        # dbt build runs each model then tests it immediately, so tests never target an unbuilt table
+        run: dbt build --profiles-dir .

.github/workflows/weekly_run.yml

@@ -0,0 +1,90 @@
+name: Weekly Production Analytics Run
+
+# ==============================================================================
+# DOCUMENTATION & PURPOSE
+# ==============================================================================
+# This workflow orchestrates the core production deployment of the Oppia
+# Analytics dbt pipeline. It runs automatically every week to refresh production
+# reporting datasets and can be triggered manually by the data team when needed.
+#
+# Environment Target: Production (oppia-analytics-prod)
+# Key Operations:
+#   1. Authenticates to GCP using an automated service account.
+#   2. Dynamically isolates database credentials to prevent local Git leaks.
+#   3. Runs data quality assertions and transforms fresh production analytics logs.
+# ==============================================================================
+
+on:
+  schedule:
+    # Runs at 00:00 UTC every Sunday (Adjust cron expression for your preferred timezone)
+    - cron: '0 0 * * 0'
+  workflow_dispatch:
+    # Allows the analytics team to trigger a one-off run manually from the GitHub UI
+
+jobs:
+  dbt_run:
+    name: Execute dbt Production Pipeline
+    runs-on: ubuntu-latest
+
+    steps:
+      # Clone the repository codebase onto the temporary GitHub virtual runner
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          # Use caching to decrease run times for subsequent runs
+          cache: 'pip'
+
+      - name: Install dbt and BigQuery Dependencies
+        run: |
+          pip install --upgrade pip
+          pip install dbt-bigquery
+
+      # Exchange GitHub secret tokens for active, authenticated Google Cloud sessions
+      - name: Authenticate to GCP Natively
+        run: |
+          # Write the secret JSON token safely as a raw literal string
+          cat << 'EOF' > ${HOME}/gcp_key.json
+          ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+          EOF
+
+          # Activate the service account using the pre-installed gcloud CLI
+          gcloud auth activate-service-account --key-file=${HOME}/gcp_key.json
+
+          # Set the application default credentials environment variable for dbt
+          echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/gcp_key.json" >> $GITHUB_ENV
+
+      # Generate a production connection profile using service-account credentials
+      - name: Create dynamic profiles.yml for dbt
+        run: |
+          cat << EOF > profiles.yml
+          oppia_analytics:
+            target: prod
+            outputs:
+              prod:
+                type: bigquery
+                method: service-account
+                keyfile: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
+                project: oppia-analytics-prod
+                dataset: analytics_production
+                threads: 4
+                timeout_seconds: 300
+                location: US
+          EOF
+
+      # Validate infrastructure paths and check that BigQuery is responding
+      - name: Verify dbt Connection
+        run: dbt debug --profiles-dir .
+
+      # Pull down any third-party open-source dbt packages listed in packages.yml
+      - name: Install dbt Packages (if any)
+        run: dbt deps --profiles-dir .
+
+      # Perform a comprehensive build (Compile, Model materialization, and Data testing)
+      - name: Execute and Test Pipeline
+        # 'dbt build' compiles, runs, and tests every staging, dim, fct, and agg table sequentially.
+        # If an upstream table fails a test, downstream tables are skipped to preserve data integrity.
+        run: dbt build --target prod --profiles-dir .

.gitignore

@@ -1,143 +1,16 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-lerna-debug.log*
+# Logs and local dbt artifacts
+logs/
+target/
+dbt_packages/
+dbt_modules/
 
-# Diagnostic reports (https://nodejs.org/api/report.html)
-report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+# Local connection profiles (Where passwords/keys live)
+profiles.yml
 
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-*.lcov
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directories
-node_modules/
-jspm_packages/
-
-# Snowpack dependency directory (https://snowpack.dev/)
-web_modules/
-
-# TypeScript cache
-*.tsbuildinfo
-
-# Optional npm cache directory
-.npm
-
-# Optional eslint cache
-.eslintcache
-
-# Optional stylelint cache
-.stylelintcache
-
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
-# dotenv environment variable files
+# OS generated files
+.DS_Store
 .env
-.env.*
-!.env.example
-
-# parcel-bundler cache (https://parceljs.org/)
-.cache
-.parcel-cache
-
-# Next.js build output
-.next
-out
-
-# Nuxt.js build / generate output
-.nuxt
-dist
-.output
-
-# Gatsby files
-.cache/
-# Comment in the public line in if your project uses Gatsby and not Next.js
-# https://nextjs.org/blog/next-9-1#public-directory-support
-# public
-
-# vuepress build output
-.vuepress/dist
-
-# vuepress v2.x temp directory
-.temp
-
-# Sveltekit cache directory
-.svelte-kit/
-
-# vitepress build output
-**/.vitepress/dist
-
-# vitepress cache directory
-**/.vitepress/cache
-
-# Docusaurus cache and generated files
-.docusaurus
-
-# Serverless directories
-.serverless/
-
-# FuseBox cache
-.fusebox/
-
-# DynamoDB Local files
-.dynamodb/
-
-# Firebase cache directory
-.firebase/
-
-# TernJS port file
-.tern-port
-
-# Stores VSCode versions used for testing VSCode extensions
-.vscode-test
-
-# pnpm
-.pnpm-store
-
-# yarn v3
-.pnp.*
-.yarn/*
-!.yarn/patches
-!.yarn/plugins
-!.yarn/releases
-!.yarn/sdks
-!.yarn/versions
 
-# Vite files
-vite.config.js.timestamp-*
-vite.config.ts.timestamp-*
-.vite/
+# Service account keys (Safety net to prevent credential leaks)
+*.json
+*.pem

README.md

@@ -0,0 +1,87 @@
+# Oppia Product Analytics Data Platform
+
+This repository houses the central dbt (Data Build Tool) transformation pipeline for Oppia. It ingests raw logging inputs from both the Web server and Android client applications and converts them into structured, performance-optimized analytical datasets inside Google Cloud BigQuery.
+
+---
+
+## Repository Topology
+
+The project uses three dbt model layers. Folder names are part of the dbt
+configuration in `dbt_project.yml`, so new models should be added to the
+corresponding layer.
+
+  ├── models/
+  │   ├── staging/
+  │   │   ├── web/             # Raw web sources and web event cleaning
+  │   │   ├── android/         # Raw Android sources and Android event cleaning
+  │   │   └── cuj_reference/   # CUJ workbook inventory and step definitions
+  │   ├── intermediate/        # Reusable transformations shared by marts
+  │   │   └── cuj_health/      # CUJ mappings, readiness, progression, metrics
+  │   └── marts/               # Business-facing models by product domain
+  │       ├── users/
+  │       ├── curriculum/
+  │       ├── growth_outreach/
+  │       └── cuj_health/      # Semantic Layer CUJ-health outputs
+  ├── seeds/cuj_health/         # Governed CUJ mappings, thresholds, and step pairs
+  ├── tests/cuj_health/         # Custom CUJ-health assertions
+  ├── macros/                  # Reusable dbt macros across all domains
+  │   ├── ga4/                 # Reusable GA4 event-parameter extraction
+  │   ├── cuj_health/          # Shared CUJ-health calculations
+  │   └── generate_surrogate_key.sql
+  ├── utils/
+  │   └── udf/                 # Warehouse user-defined functions
+  ├── dbt_project.yml           # Model routing and project scope
+  └── profiles.yml.example      # Credential configuration blueprint
+
+### Model Naming
+
+Use a double underscore between the entity and the business subject, for
+example `stg_web_analytics__events` or `int_web_cuj__event_matches`. Keep
+source definitions in `src_<platform>.yml` files and keep model descriptions
+and tests beside the models they document.
+
+---
+
+## ⚙️ Local Sandbox Environment Setup
+
+Before compiling data structures locally, developers must establish active credentials to access the development sandboxes inside `oppia-analytics-test`.
+
+### 1. Initialize Authentication and Local Dependencies
+Ensure you have Python 3.10+ installed globally, then initialize your analytics space:
+
+    # Install core database compilation tools
+    pip install dbt-bigquery
+
+    # Pull down open-source external packages
+    dbt deps
+
+### 2. Configure Your Connection Profile
+Local credentials are kept strictly out of git version control.
+
+1. Copy the tracking template:
+   cp profiles.yml.example profiles.yml
+
+2. Open your newly created `profiles.yml` file and replace "dev_yourname" with your specific developer schema signature (e.g., dev_johndoe).
+
+3. Authenticate with Google Cloud using your local user credentials:
+   gcloud auth application-default login
+
+### 3. Verify System Path Execution
+Run a diagnostic framework check to ensure dbt can establish a secure handshake with BigQuery:
+
+    dbt debug
+
+---
+
+## 🚀 Daily Execution Commands
+
+* Compile the structural SQL lineage tree:
+  dbt compile
+
+* Build data tables inside your personal schema sandbox:
+  dbt run --target dev
+
+* Execute assertion tests against data quality constraints:
+  dbt test --target dev
+
+For full details regarding the analytics architecture, query writing structures, or production merge criteria, please read the documentation inside the [Models Directory README](models/README.md).