diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml new file mode 100644 index 0000000..8671505 --- /dev/null +++ b/.github/workflows/pr_validation.yml @@ -0,0 +1,59 @@ +name: Pull Request Data Validation + +on: + pull_request: + branches: [ develop ] + +jobs: + validate_sql: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Dependencies + run: | + pip install dbt-bigquery sqlfluff-templater-dbt + + - name: Authenticate to GCP Natively + run: | + # Write the secret JSON token safely as a raw literal string + cat << 'EOF' > ${HOME}/gcp_key.json + ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + EOF + + # Activate the service account using the pre-installed gcloud CLI + gcloud auth activate-service-account --key-file=${HOME}/gcp_key.json + + # Set the application default credentials environment variable for dbt + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/gcp_key.json" >> $GITHUB_ENV + + # Generate a local connection profile using service-account credentials + - name: Create dynamic profiles.yml for dbt + run: | + cat << EOF > profiles.yml + oppia_analytics: + target: dev + outputs: + dev: + type: bigquery + method: service-account + keyfile: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} + project: oppia-analytics-test + dataset: dev_github_actions + threads: 4 + timeout_seconds: 300 + location: US + EOF + + - name: Install dbt Packages (if any) + run: dbt deps --profiles-dir . + + - name: Build and Test Models + # dbt build runs each model then tests it immediately, so tests never target an unbuilt table + run: dbt build --profiles-dir . diff --git a/.github/workflows/weekly_run.yml b/.github/workflows/weekly_run.yml new file mode 100644 index 0000000..1f5f07d --- /dev/null +++ b/.github/workflows/weekly_run.yml @@ -0,0 +1,90 @@ +name: Weekly Production Analytics Run + +# ============================================================================== +# DOCUMENTATION & PURPOSE +# ============================================================================== +# This workflow orchestrates the core production deployment of the Oppia +# Analytics dbt pipeline. It runs automatically every week to refresh production +# reporting datasets and can be triggered manually by the data team when needed. +# +# Environment Target: Production (oppia-analytics-prod) +# Key Operations: +# 1. Authenticates to GCP using an automated service account. +# 2. Dynamically isolates database credentials to prevent local Git leaks. +# 3. Runs data quality assertions and transforms fresh production analytics logs. +# ============================================================================== + +on: + schedule: + # Runs at 00:00 UTC every Sunday (Adjust cron expression for your preferred timezone) + - cron: '0 0 * * 0' + workflow_dispatch: + # Allows the analytics team to trigger a one-off run manually from the GitHub UI + +jobs: + dbt_run: + name: Execute dbt Production Pipeline + runs-on: ubuntu-latest + + steps: + # Clone the repository codebase onto the temporary GitHub virtual runner + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + # Use caching to decrease run times for subsequent runs + cache: 'pip' + + - name: Install dbt and BigQuery Dependencies + run: | + pip install --upgrade pip + pip install dbt-bigquery + + # Exchange GitHub secret tokens for active, authenticated Google Cloud sessions + - name: Authenticate to GCP Natively + run: | + # Write the secret JSON token safely as a raw literal string + cat << 'EOF' > ${HOME}/gcp_key.json + ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + EOF + + # Activate the service account using the pre-installed gcloud CLI + gcloud auth activate-service-account --key-file=${HOME}/gcp_key.json + + # Set the application default credentials environment variable for dbt + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/gcp_key.json" >> $GITHUB_ENV + + # Generate a production connection profile using service-account credentials + - name: Create dynamic profiles.yml for dbt + run: | + cat << EOF > profiles.yml + oppia_analytics: + target: prod + outputs: + prod: + type: bigquery + method: service-account + keyfile: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} + project: oppia-analytics-prod + dataset: analytics_production + threads: 4 + timeout_seconds: 300 + location: US + EOF + + # Validate infrastructure paths and check that BigQuery is responding + - name: Verify dbt Connection + run: dbt debug --profiles-dir . + + # Pull down any third-party open-source dbt packages listed in packages.yml + - name: Install dbt Packages (if any) + run: dbt deps --profiles-dir . + + # Perform a comprehensive build (Compile, Model materialization, and Data testing) + - name: Execute and Test Pipeline + # 'dbt build' compiles, runs, and tests every staging, dim, fct, and agg table sequentially. + # If an upstream table fails a test, downstream tables are skipped to preserve data integrity. + run: dbt build --target prod --profiles-dir . diff --git a/.gitignore b/.gitignore index 872d5f6..d44109f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,143 +1,16 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* +# Logs and local dbt artifacts +logs/ +target/ +dbt_packages/ +dbt_modules/ -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json +# Local connection profiles (Where passwords/keys live) +profiles.yml -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files +# OS generated files +.DS_Store .env -.env.* -!.env.example - -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist -.output - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp directory -.temp - -# Sveltekit cache directory -.svelte-kit/ - -# vitepress build output -**/.vitepress/dist - -# vitepress cache directory -**/.vitepress/cache - -# Docusaurus cache and generated files -.docusaurus - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# Firebase cache directory -.firebase/ - -# TernJS port file -.tern-port - -# Stores VSCode versions used for testing VSCode extensions -.vscode-test - -# pnpm -.pnpm-store - -# yarn v3 -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/sdks -!.yarn/versions -# Vite files -vite.config.js.timestamp-* -vite.config.ts.timestamp-* -.vite/ +# Service account keys (Safety net to prevent credential leaks) +*.json +*.pem diff --git a/README.md b/README.md new file mode 100644 index 0000000..0002f19 --- /dev/null +++ b/README.md @@ -0,0 +1,87 @@ +# Oppia Product Analytics Data Platform + +This repository houses the central dbt (Data Build Tool) transformation pipeline for Oppia. It ingests raw logging inputs from both the Web server and Android client applications and converts them into structured, performance-optimized analytical datasets inside Google Cloud BigQuery. + +--- + +## Repository Topology + +The project uses three dbt model layers. Folder names are part of the dbt +configuration in `dbt_project.yml`, so new models should be added to the +corresponding layer. + + ├── models/ + │ ├── staging/ + │ │ ├── web/ # Raw web sources and web event cleaning + │ │ ├── android/ # Raw Android sources and Android event cleaning + │ │ └── cuj_reference/ # CUJ workbook inventory and step definitions + │ ├── intermediate/ # Reusable transformations shared by marts + │ │ └── cuj_health/ # CUJ mappings, readiness, progression, metrics + │ └── marts/ # Business-facing models by product domain + │ ├── users/ + │ ├── curriculum/ + │ ├── growth_outreach/ + │ └── cuj_health/ # Semantic Layer CUJ-health outputs + ├── seeds/cuj_health/ # Governed CUJ mappings, thresholds, and step pairs + ├── tests/cuj_health/ # Custom CUJ-health assertions + ├── macros/ # Reusable dbt macros across all domains + │ ├── ga4/ # Reusable GA4 event-parameter extraction + │ ├── cuj_health/ # Shared CUJ-health calculations + │ └── generate_surrogate_key.sql + ├── utils/ + │ └── udf/ # Warehouse user-defined functions + ├── dbt_project.yml # Model routing and project scope + └── profiles.yml.example # Credential configuration blueprint + +### Model Naming + +Use a double underscore between the entity and the business subject, for +example `stg_web_analytics__events` or `int_web_cuj__event_matches`. Keep +source definitions in `src_.yml` files and keep model descriptions +and tests beside the models they document. + +--- + +## ⚙️ Local Sandbox Environment Setup + +Before compiling data structures locally, developers must establish active credentials to access the development sandboxes inside `oppia-analytics-test`. + +### 1. Initialize Authentication and Local Dependencies +Ensure you have Python 3.10+ installed globally, then initialize your analytics space: + + # Install core database compilation tools + pip install dbt-bigquery + + # Pull down open-source external packages + dbt deps + +### 2. Configure Your Connection Profile +Local credentials are kept strictly out of git version control. + +1. Copy the tracking template: + cp profiles.yml.example profiles.yml + +2. Open your newly created `profiles.yml` file and replace "dev_yourname" with your specific developer schema signature (e.g., dev_johndoe). + +3. Authenticate with Google Cloud using your local user credentials: + gcloud auth application-default login + +### 3. Verify System Path Execution +Run a diagnostic framework check to ensure dbt can establish a secure handshake with BigQuery: + + dbt debug + +--- + +## 🚀 Daily Execution Commands + +* Compile the structural SQL lineage tree: + dbt compile + +* Build data tables inside your personal schema sandbox: + dbt run --target dev + +* Execute assertion tests against data quality constraints: + dbt test --target dev + +For full details regarding the analytics architecture, query writing structures, or production merge criteria, please read the documentation inside the [Models Directory README](models/README.md). diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..8ffd8a3 --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,35 @@ +name: 'oppia_analytics' +version: '1.0.0' +config-version: 2 +profile: 'oppia_analytics' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] + +clean-targets: + - "target" + - "dbt_packages" + +# This section dynamically routes your tables based on the target (dev vs prod) +models: + oppia_analytics: + +materialized: table + + # Staging Layer configuration + # Raw server databases are configured on the source() itself, not here. + staging: + +schema: staging + +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" + + # Intermediate Layer: reusable transformations shared by marts + intermediate: + +schema: intermediate + +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" + + # Business-facing marts organized by product domain + marts: + +schema: marts + +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" diff --git a/macros/generate_surrogate_key.sql b/macros/generate_surrogate_key.sql new file mode 100644 index 0000000..7c80411 --- /dev/null +++ b/macros/generate_surrogate_key.sql @@ -0,0 +1,21 @@ +{% macro generate_surrogate_key(field_list) %} +{#- + PURPOSE: Generates a deterministic MD5 surrogate key hash across a list of columns. + COMPATIBILITY: Google Cloud BigQuery (Standard SQL) + USAGE: {{ generate_surrogate_key(['user_id', 'lesson_id']) }} AS assignment_sk +-#} +{%- if not field_list -%} + {{ exceptions.raise_compiler_error("generate_surrogate_key() requires a non-empty field_list; CONCAT() with no arguments is invalid in BigQuery.") }} +{%- endif -%} +{%- set field_expressions = [] -%} + +{%- for field in field_list -%} + {%- do field_expressions.append("COALESCE(CAST(" ~ field ~ " AS STRING), '_null_')") -%} + {%- if not loop.last -%} + {%- do field_expressions.append("'-'") -%} + {%- endif -%} +{%- endfor -%} + +TO_HEX(MD5(CONCAT({{ field_expressions | join(', ') }}))) + +{% endmacro %} diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000..1778527 --- /dev/null +++ b/models/README.md @@ -0,0 +1,126 @@ +# Oppia Transformation Layer (dbt Models) + +Welcome to the data modeling layer for the Oppia Product Analytics pipeline. This directory contains all transformation logic translating granular web and Android database event logs into analysis-ready reporting datasets. + +## Pipeline Architecture & Multi-Project Routing +Our analytics infrastructure spans multiple Google Cloud Projects (GCP) to isolate development from live production dashboards. dbt handles the routing across these environments automatically based on your execution command target. + +* **Test Environment (`oppia-analytics-test`)**: Used for local analyst development and automated Pull Request checks. Reads raw logs from the test web/Android servers and writes intermediate and mart outputs to the test project. +* **Production Environment (`oppia-analytics-prod`)**: Houses live dashboards. Reads raw logs from production Web/Android servers and writes the same model layers to the production project. + +The configured model layers are: + +1. **Staging (`staging/`)**: Source-aligned cleaning, type casting, and identity standardization for Web, Android, and CUJ-reference inputs. +2. **Intermediate (`intermediate/`)**: Reusable transformations such as unified users and CUJ-health event mapping, readiness, matching, and progression logic. +3. **Marts (`marts/`)**: Business-facing dimensions, facts, and aggregations organized by `users/`, `curriculum/`, `growth_outreach/`, and `cuj_health/`. + +CUJ-health models use the governed inputs in `seeds/cuj_health/`, custom assertions in `tests/cuj_health/`, and reusable calculations in `macros/cuj_health/`. + +--- + +## 🛠️ Analyst & Developer Workflow + +To ensure pipeline stability and prevent breaking production data structures, all contributors must strictly follow this development lifecycle. + +### Phase 1: Local Feature Development +When tasked with writing a new SQL query or editing an existing model, do not modify production files directly. + +1. **Create a Feature Branch:** Pull the latest changes from `develop` and open a local feature branch: +```bash + git checkout develop + git pull origin develop + git checkout -b feature/your-feature-name + ``` +2. **Write Pure SQL according to the Platform Skeleton:** Create your model inside the appropriate directory (e.g., `/models/staging/web/` or `/models/intermediate/cuj_health/web/`). Write your query utilizing proper CTE naming conventions, ensuring `SELECT *` is avoided in final projection blocks. + + Every dbt model script must follow this structure: +```sql + -- Project: oppia-web-analytics or oppia-android-analytics + -- Owner: analytics-team + -- Purpose: Brief single-sentence explanation of what this specific asset evaluates. + -- Note: Detailed column descriptions and data quality assertions are managed inside the corresponding schema.yml file. + + WITH source_data AS ( + SELECT * FROM {{ ref('stg_web_analytics__events') }} + ), + + lesson_progress AS ( + SELECT + user_id, + lesson_id, + progress_percent, + updated_at + FROM source_data + ) + + SELECT + user_id, + lesson_id, + progress_percent, + updated_at + FROM lesson_progress + ``` + *Never hardcode absolute dataset paths; always use the dependency tracking reference macro `{{ ref() }}` or `{{ source() }}`.* + +3. **Run Locally in Sandbox:** Execute your code using the dev target. dbt will safely direct your models to build inside your own personal sandbox dataset within the test project: +```bash + dbt run --target dev + ``` +4. **Add Schema Definitions:** Document your model and columns inside the corresponding `schema.yml` file. Define data quality tests (such as `not_null` or `unique`) on critical keys. + +### Phase 2: Pull Request & Automated Code Validation (CI) +Once your local runs compile successfully and your data quality looks accurate, it is time to move your code toward production. + +1. **Commit and Push:** Push your feature branch to GitHub: +```bash + git add . + git commit -m "feat: added web lesson completed aggregation" + git push origin feature/your-feature-name + ``` +2. **Open a Pull Request (PR):** Open a PR targeting the `develop` branch. +3. **Automated Quality Gates:** Opening the PR automatically kicks off a GitHub Actions CI pipeline. This pipeline logs into `oppia-analytics-test`, checks your SQL syntax, verifies model relationships, and builds the lineage tree. + * *If the pipeline fails, the PR will block merging until you fix the compilation or syntax errors.* +4. **Peer Review:** At least one team member must review and approve your PR before it can be merged into `develop`. + +### Phase 3: Deployment to Production +Once approved and merged into `develop`, the automation pipeline takes over. + +* **Weekly Automated Production Run:** Every Sunday at midnight UTC, a production GitHub Action wakes up, targets the `prod` configuration, and executes the updated codebase against live production data inside `oppia-analytics-prod`. +* **One-off Production Runs:** If an urgent run is required between weekly cycles, an analytics team lead can manually trigger a production run via the **Actions** tab in the GitHub UI using the `workflow_dispatch` option. + +--- + +## Data Modeling Tiers & Pipeline Execution Guarantees +All scripts across these tiers must be **strictly idempotent**. Running a pipeline or individual script multiple times must produce the exact same table state without duplicating metrics, multiplying records, or generating orphaned rows. + +1. **Staging (`staging/`)**: Source-aligned data cleaning and standardized data-type casting mapping 1:1 with source nodes. +2. **Intermediate (`intermediate/`)**: Shared transformations that prepare conformed inputs for multiple marts. +3. **Marts (`marts/`)**: Business-facing dimensions, facts, and aggregations organized by product domain. + +--- + +## ⚙️ Performance Optimization (Partitioning & Clustering) + +To optimize query performance and minimize Google Cloud BigQuery analysis costs, all high-volume tables (especially within the `fct/` and `agg/` layers) must utilize dbt configuration blocks for performance tuning: + +* **Partitioning:** Every high-volume intermediate or mart event stream must be partitioned by a date or timestamp column (e.g., `event_at` or `created_at`). This isolates queries to specific time ranges instead of scanning the entire table history. +* **Clustering:** Tables must be clustered by high-cardinality columns that are frequently used in `WHERE` filters or `JOIN` clauses (e.g., `platform`, `user_id`, `lesson_id`). + +### How to Implement This in a Model File +Analysts must add a dbt configuration block to the very top of their SQL file like this: + +```sql +{{ config( + materialized='table', + partition_by={ + "field": "event_date", + "data_type": "date", + "granularity": "day" + }, + cluster_by=["platform", "lesson_id"] +) }} + +WITH raw_data AS ( + SELECT * FROM {{ ref('stg_web_analytics__events') }} +), +... diff --git a/models/staging/web/schema.yml b/models/staging/web/schema.yml new file mode 100644 index 0000000..8026853 --- /dev/null +++ b/models/staging/web/schema.yml @@ -0,0 +1,10 @@ +version: 2 + +models: + - name: stg_web_analytics__events + description: "Standardized staging layer capturing core web log interactions." + columns: + - name: user_pseudo_id + # user_id is only populated for logged-in users; user_pseudo_id is GA4's always-present device/client identifier + tests: + - not_null diff --git a/models/staging/web/src_web.yml b/models/staging/web/src_web.yml new file mode 100644 index 0000000..9bbfe26 --- /dev/null +++ b/models/staging/web/src_web.yml @@ -0,0 +1,10 @@ +version: 2 + +sources: + - name: raw_web_server + # Dynamically switches servers based on your target (dev vs prod) + database: "{{ 'oppiaserver' if target.name == 'prod' else 'oppiatestserver' }}" + schema: "{{ 'analytics_261927573' if target.name == 'prod' else 'analytics_264617348' }}" + tables: + - name: events_* + description: "Date-sharded GA4/Firebase event export tables (events_YYYYMMDD); query via wildcard with _TABLE_SUFFIX." diff --git a/models/staging/web/stg_web_analytics__events.sql b/models/staging/web/stg_web_analytics__events.sql new file mode 100644 index 0000000..89be64a --- /dev/null +++ b/models/staging/web/stg_web_analytics__events.sql @@ -0,0 +1,44 @@ +-- Project: oppia-web-analytics +-- Owner: analytics-team +-- Purpose: Standardized staging layer capturing core web log interactions. +-- Note: Detailed column descriptions and data quality assertions are managed inside the corresponding schema.yml file. + +WITH source_data AS ( + -- events_* wildcard scans all daily shards; _TABLE_SUFFIX exposes each shard's date suffix + SELECT * FROM {{ source('raw_web_server', 'events_*') }} + WHERE _TABLE_SUFFIX BETWEEN FORMAT_DATE('%Y%m%d', DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY)) + AND FORMAT_DATE('%Y%m%d', CURRENT_DATE()) +), + +final_events AS ( + SELECT + user_id, + user_pseudo_id, + event_name, + event_date, + TIMESTAMP_MICROS(event_timestamp) AS event_timestamp, + device.category AS device_category, + device.operating_system AS operating_system, + geo.country AS country, + traffic_source.source AS traffic_source, + traffic_source.medium AS traffic_medium, + -- GA4 stores custom event attributes as key/value pairs rather than fixed columns + (SELECT value.string_value FROM UNNEST(event_params) WHERE key = 'page_location' LIMIT 1) AS page_location, + (SELECT value.int_value FROM UNNEST(event_params) WHERE key = 'ga_session_id' LIMIT 1) AS ga_session_id + FROM source_data +) + +SELECT + user_id, + user_pseudo_id, + event_name, + event_date, + event_timestamp, + device_category, + operating_system, + country, + traffic_source, + traffic_medium, + page_location, + ga_session_id +FROM final_events diff --git a/profiles.yml.example b/profiles.yml.example new file mode 100644 index 0000000..cf35d1c --- /dev/null +++ b/profiles.yml.example @@ -0,0 +1,14 @@ +# profiles.yml.example +# COPY THIS FILE, RENAME TO profiles.yml, AND DO NOT COMMIT IT TO GIT. +oppia_analytics: + target: dev + outputs: + dev: + type: bigquery + method: oauth + project: oppia-analytics-test + # Change this to your name (e.g., dev_johndoe) + dataset: "dev_yourname" + threads: 4 + timeout_seconds: 300 + location: US