From e0e0c3a42c092f4651f76dbe39ea2c10c90030f2 Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Wed, 17 Jun 2026 21:38:57 +0800 Subject: [PATCH 01/12] Initial setup of the repository --- .github/workflows/pr_validation.yml | 51 ++++++ .github/workflows/weekly_run.yml | 82 ++++++++++ .gitignore | 151 ++---------------- README.md | 0 dbt_project.yml | 41 +++++ macros/generate_surrogate_key.sql | 0 models/README.md | 119 ++++++++++++++ models/agg/README.md | 8 + models/agg/web/agg_web_lessons__completed.sql | 45 ++++++ models/dim/README.md | 8 + models/fct/README.md | 13 ++ models/schema.yml | 22 +++ models/stg/README.md | 11 ++ models/stg/web/schema.yml | 10 ++ models/stg/web/sources.yml | 10 ++ models/stg/web/stg_web_events.sql | 24 +++ profiles.yml.example | 14 ++ 17 files changed, 470 insertions(+), 139 deletions(-) create mode 100644 .github/workflows/pr_validation.yml create mode 100644 .github/workflows/weekly_run.yml create mode 100644 README.md create mode 100644 dbt_project.yml create mode 100644 macros/generate_surrogate_key.sql create mode 100644 models/README.md create mode 100644 models/agg/README.md create mode 100644 models/agg/web/agg_web_lessons__completed.sql create mode 100644 models/dim/README.md create mode 100644 models/fct/README.md create mode 100644 models/schema.yml create mode 100644 models/stg/README.md create mode 100644 models/stg/web/schema.yml create mode 100644 models/stg/web/sources.yml create mode 100644 models/stg/web/stg_web_events.sql create mode 100644 profiles.yml.example diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml new file mode 100644 index 0000000..351d76b --- /dev/null +++ b/.github/workflows/pr_validation.yml @@ -0,0 +1,51 @@ +name: Pull Request Data Validation + +on: + pull_request: + branches: [ develop ] + +jobs: + validate_sql: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install Dependencies + run: | + pip install dbt-bigquery sqlfluff-templater-dbt + + - name: Authenticate to GCP + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + # Generate a local connection profile using service-account credentials + - name: Create dynamic profiles.yml for dbt + run: | + cat << EOF > profiles.yml + oppia_analytics: + target: dev + outputs: + dev: + type: bigquery + method: service-account + keyfile: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} # Injected automatically by the auth step + project: oppia-analytics-test + dataset: dev_github_actions + threads: 4 + timeout_seconds: 300 + location: US + EOF + + - name: Test Integrity Constraints and Schema Setup + run: dbt test --profiles-dir . + + - name: Test Compilation and Lineage + # dbt compile checks syntax, relationships, and lineage macro links without running jobs on BigQuery + run: dbt compile --profiles-dir . diff --git a/.github/workflows/weekly_run.yml b/.github/workflows/weekly_run.yml new file mode 100644 index 0000000..94c7454 --- /dev/null +++ b/.github/workflows/weekly_run.yml @@ -0,0 +1,82 @@ +name: Weekly Production Analytics Run + +# ============================================================================== +# DOCUMENTATION & PURPOSE +# ============================================================================== +# This workflow orchestrates the core production deployment of the Oppia +# Analytics dbt pipeline. It runs automatically every week to refresh production +# reporting datasets and can be triggered manually by the data team when needed. +# +# Environment Target: Production (oppia-analytics-prod) +# Key Operations: +# 1. Authenticates to GCP using an automated service account. +# 2. Dynamically isolates database credentials to prevent local Git leaks. +# 3. Runs data quality assertions and transforms fresh production analytics logs. +# ============================================================================== + +on: + schedule: + # Runs at 00:00 UTC every Sunday (Adjust cron expression for your preferred timezone) + - cron: '0 0 * * 0' + workflow_dispatch: + # Allows the analytics team to trigger a one-off run manually from the GitHub UI + +jobs: + dbt_run: + name: Execute dbt Production Pipeline + runs-on: ubuntu-latest + + steps: + # Clone the repository codebase onto the temporary GitHub virtual runner + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + # Use caching to decrease run times for subsequent runs + cache: 'pip' + + - name: Install dbt and BigQuery Dependencies + run: | + pip install --upgrade pip + pip install dbt-bigquery + + # Exchange GitHub secret tokens for active, authenticated Google Cloud sessions + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + # Generate a production connection profile using service-account credentials + - name: Create dynamic profiles.yml for dbt + run: | + cat << EOF > profiles.yml + oppia_analytics: + target: prod + outputs: + prod: + type: bigquery + method: service-account + keyfile: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} + project: oppia-analytics-prod + dataset: analytics_production # Base dataset namespace + threads: 4 + timeout_seconds: 300 + location: US + EOF + + # Validate infrastructure paths and check that BigQuery is responding + - name: Verify dbt Connection + run: dbt debug --profiles-dir . + + # Pull down any third-party open-source dbt packages listed in packages.yml + - name: Install dbt Packages (if any) + run: dbt deps --profiles-dir . + + # Perform a comprehensive build (Compile, Model materialization, and Data testing) + - name: Execute and Test Pipeline + # 'dbt build' compiles, runs, and tests every staging, dim, fct, and agg table sequentially. + # If an upstream table fails a test, downstream tables are skipped to preserve data integrity. + run: dbt build --target prod --profiles-dir . diff --git a/.gitignore b/.gitignore index 872d5f6..d44109f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,143 +1,16 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* +# Logs and local dbt artifacts +logs/ +target/ +dbt_packages/ +dbt_modules/ -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json +# Local connection profiles (Where passwords/keys live) +profiles.yml -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files +# OS generated files +.DS_Store .env -.env.* -!.env.example - -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist -.output - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp directory -.temp - -# Sveltekit cache directory -.svelte-kit/ - -# vitepress build output -**/.vitepress/dist - -# vitepress cache directory -**/.vitepress/cache - -# Docusaurus cache and generated files -.docusaurus - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# Firebase cache directory -.firebase/ - -# TernJS port file -.tern-port - -# Stores VSCode versions used for testing VSCode extensions -.vscode-test - -# pnpm -.pnpm-store - -# yarn v3 -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/sdks -!.yarn/versions -# Vite files -vite.config.js.timestamp-* -vite.config.ts.timestamp-* -.vite/ +# Service account keys (Safety net to prevent credential leaks) +*.json +*.pem diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..3db6dd0 --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,41 @@ +name: 'oppia_analytics' +version: '1.0.0' +config-version: 2 +profile: 'oppia_analytics' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] + +clean-targets: + - "target" + - "dbt_packages" + +# This section dynamically routes your tables based on the target (dev vs prod) +models: + oppia_analytics: + +materialized: table + + # Staging Layer configuration + stg: + +schema: stg + web: + # Dynamically selects the test web server vs the prod web server project + +database: "{{ 'oppiaserver' if target.name == 'prod' else 'oppiatestserver' }}" + android: + # Dynamically selects the test android server vs the prod android server project + # TODO: Find the correct server names for these + +database: "{{ 'oppia-android-server-prod' if target.name == 'prod' else 'oppia-android-server-test' }}" + + # Core Modeling Layers (Writes outputs to your central analytics engines) + dim: + +schema: dim + +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" + fct: + +schema: fct + +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" + agg: + +schema: agg + +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" diff --git a/macros/generate_surrogate_key.sql b/macros/generate_surrogate_key.sql new file mode 100644 index 0000000..e69de29 diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000..993796c --- /dev/null +++ b/models/README.md @@ -0,0 +1,119 @@ +# Oppia Transformation Layer (dbt Models) + +Welcome to the data modeling layer for the Oppia Product Analytics pipeline. This directory contains all transformation logic translating granular web and Android database event logs into analysis-ready reporting datasets. + +## Pipeline Architecture & Multi-Project Routing +Our analytics infrastructure spans multiple Google Cloud Projects (GCP) to isolate development from live production dashboards. dbt handles the routing across these environments automatically based on your execution command target. + +* **Test Environment (`oppia-analytics-test`)**: Used for local analyst development and automated Pull Request checks. Reads raw logs from the test web/Android servers and outputs to `test_stg`, `test_dim`, `test_fct`, and `test_agg`. +* **Production Environment (`oppia-analytics-prod`)**: Houses live dashboards. Reads raw logs from production web/Android servers and outputs to `prod_stg`, `prod_dim`, `prod_fct`, and `prod_agg`. + +--- + +## 🛠️ Analyst & Developer Workflow + +To ensure pipeline stability and prevent breaking production data structures, all contributors must strictly follow this development lifecycle. + +### Phase 1: Local Feature Development +When tasked with writing a new SQL query or editing an existing model, do not modify production files directly. + +1. **Create a Feature Branch:** Pull the latest changes from `develop` and open a local feature branch: +```bash + git checkout develop + git pull origin develop + git checkout -b feature/your-feature-name + ``` +2. **Write Pure SQL according to the Platform Skeleton:** Create your model inside the appropriate directory (e.g., `/models/stg/web/`). Write your query utilizing proper CTE naming conventions, ensuring `SELECT *` is avoided in final projection blocks. + + Every dbt model script must follow this structure: +```sql + -- Project: oppia-web-analytics or oppia-android-analytics + -- Owner: analytics-team + -- Purpose: Brief single-sentence explanation of what this specific asset evaluates. + -- Note: Detailed column descriptions and data quality assertions are managed inside the corresponding schema.yml file. + + WITH source_data AS ( + SELECT * FROM {{ ref('stg_web_events') }} + ), + + lesson_progress AS ( + SELECT + user_id, + lesson_id, + progress_percent, + updated_at + FROM source_data + ) + + SELECT + user_id, + lesson_id, + progress_percent, + updated_at + FROM lesson_progress + ``` + *Never hardcode absolute dataset paths; always use the dependency tracking reference macro `{{ ref() }}` or `{{ source() }}`.* + +3. **Run Locally in Sandbox:** Execute your code using the dev target. dbt will safely direct your models to build inside your own personal sandbox dataset within the test project: +```bash + dbt run --target dev + ``` +4. **Add Schema Definitions:** Document your model and columns inside the corresponding `schema.yml` file. Define data quality tests (such as `not_null` or `unique`) on critical keys. + +### Phase 2: Pull Request & Automated Code Validation (CI) +Once your local runs compile successfully and your data quality looks accurate, it is time to move your code toward production. + +1. **Commit and Push:** Push your feature branch to GitHub: +```bash + git add . + git commit -m "feat: added web lesson completed aggregation" + git push origin feature/your-feature-name + ``` +2. **Open a Pull Request (PR):** Open a PR targeting the `develop` branch. +3. **Automated Quality Gates:** Opening the PR automatically kicks off a GitHub Actions CI pipeline. This pipeline logs into `oppia-analytics-test`, checks your SQL syntax, verifies model relationships, and builds the lineage tree. + * *If the pipeline fails, the PR will block merging until you fix the compilation or syntax errors.* +4. **Peer Review:** At least one team member must review and approve your PR before it can be merged into `develop`. + +### Phase 3: Deployment to Production +Once approved and merged into `develop`, the automation pipeline takes over. + +* **Weekly Automated Production Run:** Every Sunday at midnight UTC, a production GitHub Action wakes up, targets the `prod` configuration, and executes the updated codebase against live production data inside `oppia-analytics-prod`. +* **One-off Production Runs:** If an urgent run is required between weekly cycles, an analytics team lead can manually trigger a production run via the **Actions** tab in the GitHub UI using the `workflow_dispatch` option. + +--- + +## Data Modeling Tiers & Pipeline Execution Guarantees +All scripts across these tiers must be **strictly idempotent**. Running a pipeline or individual script multiple times must produce the exact same table state without duplicating metrics, multiplying records, or generating orphaned rows. + +1. **Staging (`stg/`)**: Source-aligned data cleaning and standardized data-type casting mapping 1:1 with source nodes. +2. **Dimensions (`dim/`)**: Descriptive master lookup models tracking slow-moving contextual profile properties (e.g., users, lessons). +3. **Facts (`fct/`)**: Immutable chronological event streams capturing core atomic user actions. +4. **Aggregations (`agg/`)**: High-performance, performance-optimized summary metric rollups designed directly for visualization layer connections. + +--- + +## ⚙️ Performance Optimization (Partitioning & Clustering) + +To optimize query performance and minimize Google Cloud BigQuery analysis costs, all high-volume tables (especially within the `fct/` and `agg/` layers) must utilize dbt configuration blocks for performance tuning: + +* **Partitioning:** Every transaction or event stream must be partitioned by a date or timestamp column (e.g., `event_at` or `created_at`). This isolates queries to specific time ranges instead of scanning the entire table history. +* **Clustering:** Tables must be clustered by high-cardinality columns that are frequently used in `WHERE` filters or `JOIN` clauses (e.g., `platform`, `user_id`, `lesson_id`). + +### How to Implement This in a Model File +Analysts must add a dbt configuration block to the very top of their SQL file like this: + +```sql +{{ config( + materialized='table', + partition_by={ + "field": "event_date", + "data_type": "date", + "granularity": "day" + }, + cluster_by=["platform", "lesson_id"] +) }} + +WITH raw_data AS ( + SELECT * FROM {{ ref('stg_web_events') }} +), +... diff --git a/models/agg/README.md b/models/agg/README.md new file mode 100644 index 0000000..6b48af8 --- /dev/null +++ b/models/agg/README.md @@ -0,0 +1,8 @@ +# Aggregation Layer (`agg/`) + +The Aggregation Layer contains pre-aggregated, business-ready metric rollups optimized for fast analytical consumption via dashboard systems (e.g., Looker Studio). + +### 🚨 Synchronization & Core Requirements +* **Dashboard Timestamp Rule:** To maintain complete operational clarity across Oppia teams, **every external dashboard view must prominently display a data refresh notice at the top of the report** referencing the system's runtime execution window. +* **Structural Split:** Files are explicitly isolated under `web/`, `android/`, or `core/` modules. +* **Platform Marker:** Every single model outputting from this layer must explicitly contain a populated `platform` text string column. diff --git a/models/agg/web/agg_web_lessons__completed.sql b/models/agg/web/agg_web_lessons__completed.sql new file mode 100644 index 0000000..686d4c6 --- /dev/null +++ b/models/agg/web/agg_web_lessons__completed.sql @@ -0,0 +1,45 @@ +{{ config( + materialized='table', + partition_by={ + "field": "updated_date", + "data_type": "date", + "granularity": "day" + }, + cluster_by=["user_id", "lesson_id"] +) }} + +-- Project: oppia-web-analytics +-- Owner: analytics-team +-- Purpose: Performance-optimized summary rollup tracking completed web lessons. +-- Note: Column testing and metadata descriptions are defined in models/schema.yml. + +WITH lesson_progress AS ( + SELECT + user_id, + lesson_id, + progress_percent, + updated_at, + -- Creating a safe date field for BigQuery partitioning + DATE(updated_at) AS updated_date + FROM {{ ref('stg_web_events') }} +), + +final_aggregations AS ( + SELECT + user_id, + lesson_id, + progress_percent, + updated_at, + updated_date + FROM lesson_progress + -- In a real production scenario, you would add your aggregation filters here, e.g.: + -- WHERE progress_percent = 100 +) + +SELECT + user_id, + lesson_id, + progress_percent, + updated_at, + updated_date +FROM final_aggregations diff --git a/models/dim/README.md b/models/dim/README.md new file mode 100644 index 0000000..80cf281 --- /dev/null +++ b/models/dim/README.md @@ -0,0 +1,8 @@ +# Dimension Layer (`dim/`) + +The Dimension Layer maintains descriptive master attribute reference context across the analytics workspace, mapping structural attributes like user profiles, lesson categories, and interaction objects. + +## Structural Requirements +* Optimally structured for downstream joins against transactional tables (`fct/`). +* Standardized to maintain high readability and clean categorical groupings. +* Designed to track slow-moving historical attribute properties securely. diff --git a/models/fct/README.md b/models/fct/README.md new file mode 100644 index 0000000..4bb0842 --- /dev/null +++ b/models/fct/README.md @@ -0,0 +1,13 @@ +# Fact Layer (`fct/`) + +The Fact Layer models discrete event tracking structures, system interactions, and time-series metrics. These tables capture core chronological event operations, such as session updates, answers, and interactions. + +## Subfolder Organization +* `web/`: Platform-specific structures processing web application metrics. +* `android/`: Platform-specific components handling mobile event logs. +* `core/`: Uniform models shared across both platforms where input fields align perfectly. + +## Production Design Guidelines +* Granular records represent single measurable actions. +* Joins should target primary record streams with reference lookup assets (`dim/`). +* Primary records must utilize descriptive, unique identifier labels using the format `{entity}_id` or deterministic business key hashes. diff --git a/models/schema.yml b/models/schema.yml new file mode 100644 index 0000000..e2dab68 --- /dev/null +++ b/models/schema.yml @@ -0,0 +1,22 @@ +version: 2 + +models: + - name: agg_web_lessons__completed + description: "Calculates all aggregated lesson completed data on web." + config: + labels: + project: "oppia-web-analytics" + owner: "analytics-team" + columns: + - name: user_id + description: "The unique identifier for the user." + tests: + - not_null + - name: lesson_id + description: "The unique identifier for the lesson." + tests: + - not_null + - name: updated_date + description: "The UTC date when the lesson progress was recorded. Used as the table partition key." + tests: + - not_null diff --git a/models/stg/README.md b/models/stg/README.md new file mode 100644 index 0000000..6fbb2ca --- /dev/null +++ b/models/stg/README.md @@ -0,0 +1,11 @@ +# Staging Layer (`stg/`) + +The Staging Layer acts as the operational entrance threshold for raw server logging data. It transforms raw database outputs into structurally sound datasets, removing source system anomalies before core downstream computation blocks execute. + +## Layer Strategy & Requirements +* **Schema Blueprint**: Models map 1:1 against raw source tracking data tables. +* **Logic Constraints**: Limited to clean type casting, field naming standardization, and row filters. Business calculations or multi-table joins are prohibited. +* **Identity Standardization**: Every platform event mapping model must include three identity alignment keys: + * `platform`: Explicit system label marker (`web` or `android`). + * `local_user_id`: Native alphanumeric tracking ID string unique to the source server engine. + * `global_user_id`: Consolidated cross-platform matching key using standard prefix strings (`web_12345`). diff --git a/models/stg/web/schema.yml b/models/stg/web/schema.yml new file mode 100644 index 0000000..7833bc8 --- /dev/null +++ b/models/stg/web/schema.yml @@ -0,0 +1,10 @@ +version: 2 + +models: + - name: stg_web_events + description: "Standardized staging layer capturing core web log interactions." + columns: + - name: user_id + tests: + - not_null + diff --git a/models/stg/web/sources.yml b/models/stg/web/sources.yml new file mode 100644 index 0000000..67eca2c --- /dev/null +++ b/models/stg/web/sources.yml @@ -0,0 +1,10 @@ +version: 2 + +sources: + - name: raw_web_server + # Dynamically switches servers based on your target (dev vs prod) + database: "{{ 'oppiaserver' if target.name == 'prod' else 'oppiatestserver' }}" + schema: raw_logs + tables: + - name: web_events_log + description: "Raw untransformed event streaming records from the web client application." diff --git a/models/stg/web/stg_web_events.sql b/models/stg/web/stg_web_events.sql new file mode 100644 index 0000000..3123154 --- /dev/null +++ b/models/stg/web/stg_web_events.sql @@ -0,0 +1,24 @@ +-- Project: oppia-web-analytics +-- Owner: analytics-team +-- Purpose: Standardized staging layer capturing core web log interactions. +-- Note: Detailed column descriptions and data quality assertions are managed inside the corresponding schema.yml file. + +WITH source_data AS ( + SELECT * FROM {{ source('raw_web_server', 'web_events_log') }} +), + +final_events AS ( + SELECT + user_id, + lesson_id, + progress_percent, + updated_at + FROM source_data +) + +SELECT + user_id, + lesson_id, + progress_percent, + updated_at +FROM final_events diff --git a/profiles.yml.example b/profiles.yml.example new file mode 100644 index 0000000..cf35d1c --- /dev/null +++ b/profiles.yml.example @@ -0,0 +1,14 @@ +# profiles.yml.example +# COPY THIS FILE, RENAME TO profiles.yml, AND DO NOT COMMIT IT TO GIT. +oppia_analytics: + target: dev + outputs: + dev: + type: bigquery + method: oauth + project: oppia-analytics-test + # Change this to your name (e.g., dev_johndoe) + dataset: "dev_yourname" + threads: 4 + timeout_seconds: 300 + location: US From 2abb46472a3fe57fc0523ed4000cffbd2d3d3332 Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Wed, 17 Jun 2026 21:41:19 +0800 Subject: [PATCH 02/12] Update missing files --- README.md | 19 +++++++++++++++++++ macros/generate_surrogate_key.sql | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/README.md b/README.md index e69de29..4a88c34 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,19 @@ +# Oppia Product Analytics Data Platform + +This repository houses the central dbt (Data Build Tool) transformation pipeline for Oppia. It ingests raw logging inputs from both the Web server and Android client applications and converts them into structured, performance-optimized analytical datasets inside Google Cloud BigQuery. + +--- + +## 📂 Repository Topology + +```text +├── .github/workflows/ # Automated CI/CD execution runs (PR validation & Weekly deploys) +├── macros/ # Global reusable SQL compilation modules (e.g., surrogate keys) +├── models/ # Core transformation layers +│ ├── stg/ # Staging: Source cleaning and 1:1 type casting +│ ├── dim/ # Dimensions: Contextual master reference tables +│ ├── fct/ # Facts: Immutable time-series action logs +│ └── agg/ # Aggregations: High-performance dashboard rollups +├── dbt_project.yml # Core routing configurations and project scope +└── profiles.yml.example # Blueprint for credential file configuration + diff --git a/macros/generate_surrogate_key.sql b/macros/generate_surrogate_key.sql index e69de29..2097bd6 100644 --- a/macros/generate_surrogate_key.sql +++ b/macros/generate_surrogate_key.sql @@ -0,0 +1,19 @@ +{% macro generate_surrogate_key(field_list) %} +{#- + PURPOSE: Generates a deterministic MD5 surrogate key hash across a list of columns. + COMPATIBILITY: Google Cloud BigQuery (Standard SQL) + USAGE: {{ generate_surrogate_key(['user_id', 'lesson_id']) }} AS assignment_sk +-#} +{%- set field_expressions = [] -%} + +{%- for field in field_list -%} + {%- do field_expressions.append("COALESCE(CAST(" ~ field ~ " AS STRING), '_null_')") -%} + {%- if not loop.last -%} + {%- do field_expressions.append("'-'") -%} + {%- endif -%} +{%- endfor -%} + +TO_HEX(MD5(CONCAT({{ field_expressions | join(', ') }}))) + +{% endmacro %} + From 042ed44b6a46c640ccee2fd935ece4c9acba021b Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Wed, 17 Jun 2026 21:57:15 +0800 Subject: [PATCH 03/12] fix auth, update readme --- .github/workflows/pr_validation.yml | 18 +++++--- .github/workflows/weekly_run.yml | 16 +++++--- README.md | 64 ++++++++++++++++++++++++----- 3 files changed, 77 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index 351d76b..a14aeec 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -20,12 +20,18 @@ jobs: run: | pip install dbt-bigquery sqlfluff-templater-dbt - - name: Authenticate to GCP - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + - name: Authenticate to GCP Natively + run: | + # Write the secret JSON token to a temporary file on the runner + echo "${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}" > ${HOME}/gcp_key.json + + # Activate the service account using the pre-installed gcloud CLI + gcloud auth activate-service-account --key-file=${HOME}/gcp_key.json + + # Set the application default credentials environment variable for dbt + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/gcp_key.json" >> $GITHUB_ENV - # Generate a local connection profile using service-account credentials + # Generate a local connection profile using service-account credentials - name: Create dynamic profiles.yml for dbt run: | cat << EOF > profiles.yml @@ -35,7 +41,7 @@ jobs: dev: type: bigquery method: service-account - keyfile: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} # Injected automatically by the auth step + keyfile: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} project: oppia-analytics-test dataset: dev_github_actions threads: 4 diff --git a/.github/workflows/weekly_run.yml b/.github/workflows/weekly_run.yml index 94c7454..72deab7 100644 --- a/.github/workflows/weekly_run.yml +++ b/.github/workflows/weekly_run.yml @@ -44,10 +44,16 @@ jobs: pip install dbt-bigquery # Exchange GitHub secret tokens for active, authenticated Google Cloud sessions - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + - name: Authenticate to GCP Natively + run: | + # Write the secret JSON token to a temporary file on the runner + echo "${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}" > ${HOME}/gcp_key.json + + # Activate the service account using the pre-installed gcloud CLI + gcloud auth activate-service-account --key-file=${HOME}/gcp_key.json + + # Set the application default credentials environment variable for dbt + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/gcp_key.json" >> $GITHUB_ENV # Generate a production connection profile using service-account credentials - name: Create dynamic profiles.yml for dbt @@ -61,7 +67,7 @@ jobs: method: service-account keyfile: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} project: oppia-analytics-prod - dataset: analytics_production # Base dataset namespace + dataset: analytics_production threads: 4 timeout_seconds: 300 location: US diff --git a/README.md b/README.md index 4a88c34..03f56fd 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,58 @@ This repository houses the central dbt (Data Build Tool) transformation pipeline ## 📂 Repository Topology -```text -├── .github/workflows/ # Automated CI/CD execution runs (PR validation & Weekly deploys) -├── macros/ # Global reusable SQL compilation modules (e.g., surrogate keys) -├── models/ # Core transformation layers -│ ├── stg/ # Staging: Source cleaning and 1:1 type casting -│ ├── dim/ # Dimensions: Contextual master reference tables -│ ├── fct/ # Facts: Immutable time-series action logs -│ └── agg/ # Aggregations: High-performance dashboard rollups -├── dbt_project.yml # Core routing configurations and project scope -└── profiles.yml.example # Blueprint for credential file configuration + ├── .github/workflows/ # Automated CI/CD execution runs (PR validation & Weekly deploys) + ├── macros/ # Global reusable SQL compilation modules (e.g., surrogate keys) + ├── models/ # Core transformation layers + │ ├── stg/ # Staging: Source cleaning and 1:1 type casting + │ ├── dim/ # Dimensions: Contextual master reference tables + │ ├── fct/ # Facts: Immutable time-series action logs + │ └── agg/ # Aggregations: High-performance dashboard rollups + ├── dbt_project.yml # Core routing configurations and project scope + └── profiles.yml.example # Blueprint for credential file configuration +--- + +## ⚙️ Local Sandbox Environment Setup + +Before compiling data structures locally, developers must establish active credentials to access the development sandboxes inside `oppia-analytics-test`. + +### 1. Initialize Authentication and Local Dependencies +Ensure you have Python 3.10+ installed globally, then initialize your analytics space: + + # Install core database compilation tools + pip install dbt-bigquery + + # Pull down open-source external packages + dbt deps + +### 2. Configure Your Connection Profile +Local credentials are kept strictly out of git version control. + +1. Copy the tracking template: + cp profiles.yml.example profiles.yml + +2. Open your newly created `profiles.yml` file and replace "dev_yourname" with your specific developer schema signature (e.g., dev_johndoe). + +3. Authenticate with Google Cloud using your local user credentials: + gcloud auth application-default login + +### 3. Verify System Path Execution +Run a diagnostic framework check to ensure dbt can establish a secure handshake with BigQuery: + + dbt debug + +--- + +## 🚀 Daily Execution Commands + +* Compile the structural SQL lineage tree: + dbt compile + +* Build data tables inside your personal schema sandbox: + dbt run --target dev + +* Execute assertion tests against data quality constraints: + dbt test --target dev + +For full details regarding the analytics architecture, query writing structures, or production merge criteria, please read the documentation inside the [Models Directory README](models/README.md). From 84e5b761446e16f6b018444f2753282a5d2e995f Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Wed, 17 Jun 2026 22:00:32 +0800 Subject: [PATCH 04/12] fix versions --- .github/workflows/pr_validation.yml | 4 ++-- .github/workflows/weekly_run.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index a14aeec..3582b06 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -9,10 +9,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v4 + uses: actions/checkout@v4.2.2 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v5.4.0 with: python-version: '3.10' diff --git a/.github/workflows/weekly_run.yml b/.github/workflows/weekly_run.yml index 72deab7..518774b 100644 --- a/.github/workflows/weekly_run.yml +++ b/.github/workflows/weekly_run.yml @@ -29,10 +29,10 @@ jobs: steps: # Clone the repository codebase onto the temporary GitHub virtual runner - name: Checkout Code - uses: actions/checkout@v4 + uses: actions/checkout@v4.2.2 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v5.4.0 with: python-version: '3.10' # Use caching to decrease run times for subsequent runs From eae748de8826a5a58a2f87f473c1e446289d47ca Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Wed, 17 Jun 2026 22:04:37 +0800 Subject: [PATCH 05/12] update --- .github/workflows/pr_validation.yml | 6 ++++-- .github/workflows/weekly_run.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index 3582b06..40ad41a 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -22,8 +22,10 @@ jobs: - name: Authenticate to GCP Natively run: | - # Write the secret JSON token to a temporary file on the runner - echo "${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}" > ${HOME}/gcp_key.json + # Write the secret JSON token safely as a raw literal string + cat << 'EOF' > ${HOME}/gcp_key.json + ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + EOF # Activate the service account using the pre-installed gcloud CLI gcloud auth activate-service-account --key-file=${HOME}/gcp_key.json diff --git a/.github/workflows/weekly_run.yml b/.github/workflows/weekly_run.yml index 518774b..fcfcc4c 100644 --- a/.github/workflows/weekly_run.yml +++ b/.github/workflows/weekly_run.yml @@ -46,8 +46,10 @@ jobs: # Exchange GitHub secret tokens for active, authenticated Google Cloud sessions - name: Authenticate to GCP Natively run: | - # Write the secret JSON token to a temporary file on the runner - echo "${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}" > ${HOME}/gcp_key.json + # Write the secret JSON token safely as a raw literal string + cat << 'EOF' > ${HOME}/gcp_key.json + ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + EOF # Activate the service account using the pre-installed gcloud CLI gcloud auth activate-service-account --key-file=${HOME}/gcp_key.json From a8dc7ab33bf6e63b2cd89131fa6928983764d945 Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Sat, 15 Aug 2026 16:58:49 +0800 Subject: [PATCH 06/12] Update to new file tree structure --- .github/workflows/pr_validation.yml | 6 +-- .github/workflows/weekly_run.yml | 6 +-- README.md | 46 ++++++++++++++----- dbt_project.yml | 19 ++++---- macros/generate_surrogate_key.sql | 3 +- models/README.md | 27 +++++++---- models/agg/README.md | 8 ---- models/dim/README.md | 8 ---- models/fct/README.md | 13 ------ .../curriculum/_curriculum.yml} | 8 ++-- .../agg_lesson_completion_weekly.sql} | 11 ++--- .../web/schema.yml => staging/web/_web.yml} | 5 +- .../sources.yml => staging/web/src_web.yml} | 2 +- .../web/stg_web_analytics__events.sql} | 2 +- models/stg/README.md | 11 ----- 15 files changed, 80 insertions(+), 95 deletions(-) delete mode 100644 models/agg/README.md delete mode 100644 models/dim/README.md delete mode 100644 models/fct/README.md rename models/{schema.yml => marts/curriculum/_curriculum.yml} (61%) rename models/{agg/web/agg_web_lessons__completed.sql => marts/curriculum/agg_lesson_completion_weekly.sql} (67%) rename models/{stg/web/schema.yml => staging/web/_web.yml} (73%) rename models/{stg/web/sources.yml => staging/web/src_web.yml} (90%) rename models/{stg/web/stg_web_events.sql => staging/web/stg_web_analytics__events.sql} (96%) delete mode 100644 models/stg/README.md diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index 40ad41a..c2eacc5 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -9,12 +9,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v5.4.0 + uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - name: Install Dependencies run: | diff --git a/.github/workflows/weekly_run.yml b/.github/workflows/weekly_run.yml index fcfcc4c..1f5f07d 100644 --- a/.github/workflows/weekly_run.yml +++ b/.github/workflows/weekly_run.yml @@ -29,12 +29,12 @@ jobs: steps: # Clone the repository codebase onto the temporary GitHub virtual runner - name: Checkout Code - uses: actions/checkout@v4.2.2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v5.4.0 + uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' # Use caching to decrease run times for subsequent runs cache: 'pip' diff --git a/README.md b/README.md index 03f56fd..0002f19 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,41 @@ This repository houses the central dbt (Data Build Tool) transformation pipeline --- -## 📂 Repository Topology - - ├── .github/workflows/ # Automated CI/CD execution runs (PR validation & Weekly deploys) - ├── macros/ # Global reusable SQL compilation modules (e.g., surrogate keys) - ├── models/ # Core transformation layers - │ ├── stg/ # Staging: Source cleaning and 1:1 type casting - │ ├── dim/ # Dimensions: Contextual master reference tables - │ ├── fct/ # Facts: Immutable time-series action logs - │ └── agg/ # Aggregations: High-performance dashboard rollups - ├── dbt_project.yml # Core routing configurations and project scope - └── profiles.yml.example # Blueprint for credential file configuration +## Repository Topology + +The project uses three dbt model layers. Folder names are part of the dbt +configuration in `dbt_project.yml`, so new models should be added to the +corresponding layer. + + ├── models/ + │ ├── staging/ + │ │ ├── web/ # Raw web sources and web event cleaning + │ │ ├── android/ # Raw Android sources and Android event cleaning + │ │ └── cuj_reference/ # CUJ workbook inventory and step definitions + │ ├── intermediate/ # Reusable transformations shared by marts + │ │ └── cuj_health/ # CUJ mappings, readiness, progression, metrics + │ └── marts/ # Business-facing models by product domain + │ ├── users/ + │ ├── curriculum/ + │ ├── growth_outreach/ + │ └── cuj_health/ # Semantic Layer CUJ-health outputs + ├── seeds/cuj_health/ # Governed CUJ mappings, thresholds, and step pairs + ├── tests/cuj_health/ # Custom CUJ-health assertions + ├── macros/ # Reusable dbt macros across all domains + │ ├── ga4/ # Reusable GA4 event-parameter extraction + │ ├── cuj_health/ # Shared CUJ-health calculations + │ └── generate_surrogate_key.sql + ├── utils/ + │ └── udf/ # Warehouse user-defined functions + ├── dbt_project.yml # Model routing and project scope + └── profiles.yml.example # Credential configuration blueprint + +### Model Naming + +Use a double underscore between the entity and the business subject, for +example `stg_web_analytics__events` or `int_web_cuj__event_matches`. Keep +source definitions in `src_.yml` files and keep model descriptions +and tests beside the models they document. --- diff --git a/dbt_project.yml b/dbt_project.yml index 3db6dd0..d7a0b15 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -19,8 +19,8 @@ models: +materialized: table # Staging Layer configuration - stg: - +schema: stg + staging: + +schema: staging web: # Dynamically selects the test web server vs the prod web server project +database: "{{ 'oppiaserver' if target.name == 'prod' else 'oppiatestserver' }}" @@ -29,13 +29,12 @@ models: # TODO: Find the correct server names for these +database: "{{ 'oppia-android-server-prod' if target.name == 'prod' else 'oppia-android-server-test' }}" - # Core Modeling Layers (Writes outputs to your central analytics engines) - dim: - +schema: dim + # Intermediate Layer: reusable transformations shared by marts + intermediate: + +schema: intermediate +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" - fct: - +schema: fct - +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" - agg: - +schema: agg + + # Business-facing marts organized by product domain + marts: + +schema: marts +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" diff --git a/macros/generate_surrogate_key.sql b/macros/generate_surrogate_key.sql index 2097bd6..85f6831 100644 --- a/macros/generate_surrogate_key.sql +++ b/macros/generate_surrogate_key.sql @@ -15,5 +15,4 @@ TO_HEX(MD5(CONCAT({{ field_expressions | join(', ') }}))) -{% endmacro %} - +{% endmacro %} \ No newline at end of file diff --git a/models/README.md b/models/README.md index 993796c..1778527 100644 --- a/models/README.md +++ b/models/README.md @@ -5,8 +5,16 @@ Welcome to the data modeling layer for the Oppia Product Analytics pipeline. Thi ## Pipeline Architecture & Multi-Project Routing Our analytics infrastructure spans multiple Google Cloud Projects (GCP) to isolate development from live production dashboards. dbt handles the routing across these environments automatically based on your execution command target. -* **Test Environment (`oppia-analytics-test`)**: Used for local analyst development and automated Pull Request checks. Reads raw logs from the test web/Android servers and outputs to `test_stg`, `test_dim`, `test_fct`, and `test_agg`. -* **Production Environment (`oppia-analytics-prod`)**: Houses live dashboards. Reads raw logs from production web/Android servers and outputs to `prod_stg`, `prod_dim`, `prod_fct`, and `prod_agg`. +* **Test Environment (`oppia-analytics-test`)**: Used for local analyst development and automated Pull Request checks. Reads raw logs from the test web/Android servers and writes intermediate and mart outputs to the test project. +* **Production Environment (`oppia-analytics-prod`)**: Houses live dashboards. Reads raw logs from production Web/Android servers and writes the same model layers to the production project. + +The configured model layers are: + +1. **Staging (`staging/`)**: Source-aligned cleaning, type casting, and identity standardization for Web, Android, and CUJ-reference inputs. +2. **Intermediate (`intermediate/`)**: Reusable transformations such as unified users and CUJ-health event mapping, readiness, matching, and progression logic. +3. **Marts (`marts/`)**: Business-facing dimensions, facts, and aggregations organized by `users/`, `curriculum/`, `growth_outreach/`, and `cuj_health/`. + +CUJ-health models use the governed inputs in `seeds/cuj_health/`, custom assertions in `tests/cuj_health/`, and reusable calculations in `macros/cuj_health/`. --- @@ -23,7 +31,7 @@ When tasked with writing a new SQL query or editing an existing model, do not mo git pull origin develop git checkout -b feature/your-feature-name ``` -2. **Write Pure SQL according to the Platform Skeleton:** Create your model inside the appropriate directory (e.g., `/models/stg/web/`). Write your query utilizing proper CTE naming conventions, ensuring `SELECT *` is avoided in final projection blocks. +2. **Write Pure SQL according to the Platform Skeleton:** Create your model inside the appropriate directory (e.g., `/models/staging/web/` or `/models/intermediate/cuj_health/web/`). Write your query utilizing proper CTE naming conventions, ensuring `SELECT *` is avoided in final projection blocks. Every dbt model script must follow this structure: ```sql @@ -33,7 +41,7 @@ When tasked with writing a new SQL query or editing an existing model, do not mo -- Note: Detailed column descriptions and data quality assertions are managed inside the corresponding schema.yml file. WITH source_data AS ( - SELECT * FROM {{ ref('stg_web_events') }} + SELECT * FROM {{ ref('stg_web_analytics__events') }} ), lesson_progress AS ( @@ -85,10 +93,9 @@ Once approved and merged into `develop`, the automation pipeline takes over. ## Data Modeling Tiers & Pipeline Execution Guarantees All scripts across these tiers must be **strictly idempotent**. Running a pipeline or individual script multiple times must produce the exact same table state without duplicating metrics, multiplying records, or generating orphaned rows. -1. **Staging (`stg/`)**: Source-aligned data cleaning and standardized data-type casting mapping 1:1 with source nodes. -2. **Dimensions (`dim/`)**: Descriptive master lookup models tracking slow-moving contextual profile properties (e.g., users, lessons). -3. **Facts (`fct/`)**: Immutable chronological event streams capturing core atomic user actions. -4. **Aggregations (`agg/`)**: High-performance, performance-optimized summary metric rollups designed directly for visualization layer connections. +1. **Staging (`staging/`)**: Source-aligned data cleaning and standardized data-type casting mapping 1:1 with source nodes. +2. **Intermediate (`intermediate/`)**: Shared transformations that prepare conformed inputs for multiple marts. +3. **Marts (`marts/`)**: Business-facing dimensions, facts, and aggregations organized by product domain. --- @@ -96,7 +103,7 @@ All scripts across these tiers must be **strictly idempotent**. Running a pipeli To optimize query performance and minimize Google Cloud BigQuery analysis costs, all high-volume tables (especially within the `fct/` and `agg/` layers) must utilize dbt configuration blocks for performance tuning: -* **Partitioning:** Every transaction or event stream must be partitioned by a date or timestamp column (e.g., `event_at` or `created_at`). This isolates queries to specific time ranges instead of scanning the entire table history. +* **Partitioning:** Every high-volume intermediate or mart event stream must be partitioned by a date or timestamp column (e.g., `event_at` or `created_at`). This isolates queries to specific time ranges instead of scanning the entire table history. * **Clustering:** Tables must be clustered by high-cardinality columns that are frequently used in `WHERE` filters or `JOIN` clauses (e.g., `platform`, `user_id`, `lesson_id`). ### How to Implement This in a Model File @@ -114,6 +121,6 @@ Analysts must add a dbt configuration block to the very top of their SQL file li ) }} WITH raw_data AS ( - SELECT * FROM {{ ref('stg_web_events') }} + SELECT * FROM {{ ref('stg_web_analytics__events') }} ), ... diff --git a/models/agg/README.md b/models/agg/README.md deleted file mode 100644 index 6b48af8..0000000 --- a/models/agg/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Aggregation Layer (`agg/`) - -The Aggregation Layer contains pre-aggregated, business-ready metric rollups optimized for fast analytical consumption via dashboard systems (e.g., Looker Studio). - -### 🚨 Synchronization & Core Requirements -* **Dashboard Timestamp Rule:** To maintain complete operational clarity across Oppia teams, **every external dashboard view must prominently display a data refresh notice at the top of the report** referencing the system's runtime execution window. -* **Structural Split:** Files are explicitly isolated under `web/`, `android/`, or `core/` modules. -* **Platform Marker:** Every single model outputting from this layer must explicitly contain a populated `platform` text string column. diff --git a/models/dim/README.md b/models/dim/README.md deleted file mode 100644 index 80cf281..0000000 --- a/models/dim/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Dimension Layer (`dim/`) - -The Dimension Layer maintains descriptive master attribute reference context across the analytics workspace, mapping structural attributes like user profiles, lesson categories, and interaction objects. - -## Structural Requirements -* Optimally structured for downstream joins against transactional tables (`fct/`). -* Standardized to maintain high readability and clean categorical groupings. -* Designed to track slow-moving historical attribute properties securely. diff --git a/models/fct/README.md b/models/fct/README.md deleted file mode 100644 index 4bb0842..0000000 --- a/models/fct/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Fact Layer (`fct/`) - -The Fact Layer models discrete event tracking structures, system interactions, and time-series metrics. These tables capture core chronological event operations, such as session updates, answers, and interactions. - -## Subfolder Organization -* `web/`: Platform-specific structures processing web application metrics. -* `android/`: Platform-specific components handling mobile event logs. -* `core/`: Uniform models shared across both platforms where input fields align perfectly. - -## Production Design Guidelines -* Granular records represent single measurable actions. -* Joins should target primary record streams with reference lookup assets (`dim/`). -* Primary records must utilize descriptive, unique identifier labels using the format `{entity}_id` or deterministic business key hashes. diff --git a/models/schema.yml b/models/marts/curriculum/_curriculum.yml similarity index 61% rename from models/schema.yml rename to models/marts/curriculum/_curriculum.yml index e2dab68..23ab10b 100644 --- a/models/schema.yml +++ b/models/marts/curriculum/_curriculum.yml @@ -1,8 +1,8 @@ version: 2 models: - - name: agg_web_lessons__completed - description: "Calculates all aggregated lesson completed data on web." + - name: agg_lesson_completion_weekly + description: "Calculates weekly lesson completion metrics for the curriculum domain." config: labels: project: "oppia-web-analytics" @@ -17,6 +17,6 @@ models: tests: - not_null - name: updated_date - description: "The UTC date when the lesson progress was recorded. Used as the table partition key." + description: "The UTC date when lesson progress was recorded. Used as the table partition key." tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/agg/web/agg_web_lessons__completed.sql b/models/marts/curriculum/agg_lesson_completion_weekly.sql similarity index 67% rename from models/agg/web/agg_web_lessons__completed.sql rename to models/marts/curriculum/agg_lesson_completion_weekly.sql index 686d4c6..c885771 100644 --- a/models/agg/web/agg_web_lessons__completed.sql +++ b/models/marts/curriculum/agg_lesson_completion_weekly.sql @@ -10,8 +10,8 @@ -- Project: oppia-web-analytics -- Owner: analytics-team --- Purpose: Performance-optimized summary rollup tracking completed web lessons. --- Note: Column testing and metadata descriptions are defined in models/schema.yml. +-- Purpose: Performance-optimized weekly lesson completion rollup. +-- Note: Column testing and metadata descriptions are defined in _curriculum.yml. WITH lesson_progress AS ( SELECT @@ -19,9 +19,8 @@ WITH lesson_progress AS ( lesson_id, progress_percent, updated_at, - -- Creating a safe date field for BigQuery partitioning DATE(updated_at) AS updated_date - FROM {{ ref('stg_web_events') }} + FROM {{ ref('stg_web_analytics__events') }} ), final_aggregations AS ( @@ -32,8 +31,6 @@ final_aggregations AS ( updated_at, updated_date FROM lesson_progress - -- In a real production scenario, you would add your aggregation filters here, e.g.: - -- WHERE progress_percent = 100 ) SELECT @@ -42,4 +39,4 @@ SELECT progress_percent, updated_at, updated_date -FROM final_aggregations +FROM final_aggregations \ No newline at end of file diff --git a/models/stg/web/schema.yml b/models/staging/web/_web.yml similarity index 73% rename from models/stg/web/schema.yml rename to models/staging/web/_web.yml index 7833bc8..be081b1 100644 --- a/models/stg/web/schema.yml +++ b/models/staging/web/_web.yml @@ -1,10 +1,9 @@ version: 2 models: - - name: stg_web_events + - name: stg_web_analytics__events description: "Standardized staging layer capturing core web log interactions." columns: - name: user_id tests: - - not_null - + - not_null \ No newline at end of file diff --git a/models/stg/web/sources.yml b/models/staging/web/src_web.yml similarity index 90% rename from models/stg/web/sources.yml rename to models/staging/web/src_web.yml index 67eca2c..5568bdf 100644 --- a/models/stg/web/sources.yml +++ b/models/staging/web/src_web.yml @@ -7,4 +7,4 @@ sources: schema: raw_logs tables: - name: web_events_log - description: "Raw untransformed event streaming records from the web client application." + description: "Raw untransformed event streaming records from the web client application." \ No newline at end of file diff --git a/models/stg/web/stg_web_events.sql b/models/staging/web/stg_web_analytics__events.sql similarity index 96% rename from models/stg/web/stg_web_events.sql rename to models/staging/web/stg_web_analytics__events.sql index 3123154..feda96e 100644 --- a/models/stg/web/stg_web_events.sql +++ b/models/staging/web/stg_web_analytics__events.sql @@ -21,4 +21,4 @@ SELECT lesson_id, progress_percent, updated_at -FROM final_events +FROM final_events \ No newline at end of file diff --git a/models/stg/README.md b/models/stg/README.md deleted file mode 100644 index 6fbb2ca..0000000 --- a/models/stg/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Staging Layer (`stg/`) - -The Staging Layer acts as the operational entrance threshold for raw server logging data. It transforms raw database outputs into structurally sound datasets, removing source system anomalies before core downstream computation blocks execute. - -## Layer Strategy & Requirements -* **Schema Blueprint**: Models map 1:1 against raw source tracking data tables. -* **Logic Constraints**: Limited to clean type casting, field naming standardization, and row filters. Business calculations or multi-table joins are prohibited. -* **Identity Standardization**: Every platform event mapping model must include three identity alignment keys: - * `platform`: Explicit system label marker (`web` or `android`). - * `local_user_id`: Native alphanumeric tracking ID string unique to the source server engine. - * `global_user_id`: Consolidated cross-platform matching key using standard prefix strings (`web_12345`). From 63ebefd59eee056e6cda1bd6045527106b2a7272 Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Sat, 15 Aug 2026 17:04:55 +0800 Subject: [PATCH 07/12] Cleanup --- macros/generate_surrogate_key.sql | 2 +- models/marts/curriculum/_curriculum.yml | 22 ---------- .../agg_lesson_completion_weekly.sql | 42 ------------------- models/staging/web/{_web.yml => schema.yml} | 2 +- models/staging/web/src_web.yml | 2 +- .../staging/web/stg_web_analytics__events.sql | 2 +- 6 files changed, 4 insertions(+), 68 deletions(-) delete mode 100644 models/marts/curriculum/_curriculum.yml delete mode 100644 models/marts/curriculum/agg_lesson_completion_weekly.sql rename models/staging/web/{_web.yml => schema.yml} (90%) diff --git a/macros/generate_surrogate_key.sql b/macros/generate_surrogate_key.sql index 85f6831..599ff6f 100644 --- a/macros/generate_surrogate_key.sql +++ b/macros/generate_surrogate_key.sql @@ -15,4 +15,4 @@ TO_HEX(MD5(CONCAT({{ field_expressions | join(', ') }}))) -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/models/marts/curriculum/_curriculum.yml b/models/marts/curriculum/_curriculum.yml deleted file mode 100644 index 23ab10b..0000000 --- a/models/marts/curriculum/_curriculum.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 2 - -models: - - name: agg_lesson_completion_weekly - description: "Calculates weekly lesson completion metrics for the curriculum domain." - config: - labels: - project: "oppia-web-analytics" - owner: "analytics-team" - columns: - - name: user_id - description: "The unique identifier for the user." - tests: - - not_null - - name: lesson_id - description: "The unique identifier for the lesson." - tests: - - not_null - - name: updated_date - description: "The UTC date when lesson progress was recorded. Used as the table partition key." - tests: - - not_null \ No newline at end of file diff --git a/models/marts/curriculum/agg_lesson_completion_weekly.sql b/models/marts/curriculum/agg_lesson_completion_weekly.sql deleted file mode 100644 index c885771..0000000 --- a/models/marts/curriculum/agg_lesson_completion_weekly.sql +++ /dev/null @@ -1,42 +0,0 @@ -{{ config( - materialized='table', - partition_by={ - "field": "updated_date", - "data_type": "date", - "granularity": "day" - }, - cluster_by=["user_id", "lesson_id"] -) }} - --- Project: oppia-web-analytics --- Owner: analytics-team --- Purpose: Performance-optimized weekly lesson completion rollup. --- Note: Column testing and metadata descriptions are defined in _curriculum.yml. - -WITH lesson_progress AS ( - SELECT - user_id, - lesson_id, - progress_percent, - updated_at, - DATE(updated_at) AS updated_date - FROM {{ ref('stg_web_analytics__events') }} -), - -final_aggregations AS ( - SELECT - user_id, - lesson_id, - progress_percent, - updated_at, - updated_date - FROM lesson_progress -) - -SELECT - user_id, - lesson_id, - progress_percent, - updated_at, - updated_date -FROM final_aggregations \ No newline at end of file diff --git a/models/staging/web/_web.yml b/models/staging/web/schema.yml similarity index 90% rename from models/staging/web/_web.yml rename to models/staging/web/schema.yml index be081b1..a21e37b 100644 --- a/models/staging/web/_web.yml +++ b/models/staging/web/schema.yml @@ -6,4 +6,4 @@ models: columns: - name: user_id tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/staging/web/src_web.yml b/models/staging/web/src_web.yml index 5568bdf..67eca2c 100644 --- a/models/staging/web/src_web.yml +++ b/models/staging/web/src_web.yml @@ -7,4 +7,4 @@ sources: schema: raw_logs tables: - name: web_events_log - description: "Raw untransformed event streaming records from the web client application." \ No newline at end of file + description: "Raw untransformed event streaming records from the web client application." diff --git a/models/staging/web/stg_web_analytics__events.sql b/models/staging/web/stg_web_analytics__events.sql index feda96e..3123154 100644 --- a/models/staging/web/stg_web_analytics__events.sql +++ b/models/staging/web/stg_web_analytics__events.sql @@ -21,4 +21,4 @@ SELECT lesson_id, progress_percent, updated_at -FROM final_events \ No newline at end of file +FROM final_events From 81e79aa244befd1ad9311863ff2a6bee4aafbcba Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Sat, 15 Aug 2026 17:21:01 +0800 Subject: [PATCH 08/12] change staging db --- dbt_project.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index d7a0b15..8ffd8a3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -19,15 +19,10 @@ models: +materialized: table # Staging Layer configuration + # Raw server databases are configured on the source() itself, not here. staging: +schema: staging - web: - # Dynamically selects the test web server vs the prod web server project - +database: "{{ 'oppiaserver' if target.name == 'prod' else 'oppiatestserver' }}" - android: - # Dynamically selects the test android server vs the prod android server project - # TODO: Find the correct server names for these - +database: "{{ 'oppia-android-server-prod' if target.name == 'prod' else 'oppia-android-server-test' }}" + +database: "{{ 'oppia-analytics-prod' if target.name == 'prod' else 'oppia-analytics-test' }}" # Intermediate Layer: reusable transformations shared by marts intermediate: From e68a13cee29ab81d24971772d1c0b32b5c8607db Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Sat, 15 Aug 2026 17:31:39 +0800 Subject: [PATCH 09/12] Use dbt build --- .github/workflows/pr_validation.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index c2eacc5..adee48b 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -51,9 +51,6 @@ jobs: location: US EOF - - name: Test Integrity Constraints and Schema Setup - run: dbt test --profiles-dir . - - - name: Test Compilation and Lineage - # dbt compile checks syntax, relationships, and lineage macro links without running jobs on BigQuery - run: dbt compile --profiles-dir . + - name: Build and Test Models + # dbt build runs each model then tests it immediately, so tests never target an unbuilt table + run: dbt build --profiles-dir . From c128fe2dbd012f938cfb5dae8f12ecd79f5bd8c7 Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Sat, 15 Aug 2026 23:59:01 +0800 Subject: [PATCH 10/12] fix dataset name --- models/staging/web/src_web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/web/src_web.yml b/models/staging/web/src_web.yml index 67eca2c..6323911 100644 --- a/models/staging/web/src_web.yml +++ b/models/staging/web/src_web.yml @@ -4,7 +4,7 @@ sources: - name: raw_web_server # Dynamically switches servers based on your target (dev vs prod) database: "{{ 'oppiaserver' if target.name == 'prod' else 'oppiatestserver' }}" - schema: raw_logs + schema: "{{ 'analytics_261927573' if target.name == 'prod' else 'analytics_264617348' }}" tables: - name: web_events_log description: "Raw untransformed event streaming records from the web client application." From f6186c7ea717b981b25bcc8061f09fe20cb2b13f Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Sun, 16 Aug 2026 00:53:57 +0800 Subject: [PATCH 11/12] Try to get presubmit working --- models/staging/web/schema.yml | 3 +- models/staging/web/src_web.yml | 4 +-- .../staging/web/stg_web_analytics__events.sql | 34 +++++++++++++++---- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/models/staging/web/schema.yml b/models/staging/web/schema.yml index a21e37b..8026853 100644 --- a/models/staging/web/schema.yml +++ b/models/staging/web/schema.yml @@ -4,6 +4,7 @@ models: - name: stg_web_analytics__events description: "Standardized staging layer capturing core web log interactions." columns: - - name: user_id + - name: user_pseudo_id + # user_id is only populated for logged-in users; user_pseudo_id is GA4's always-present device/client identifier tests: - not_null diff --git a/models/staging/web/src_web.yml b/models/staging/web/src_web.yml index 6323911..9bbfe26 100644 --- a/models/staging/web/src_web.yml +++ b/models/staging/web/src_web.yml @@ -6,5 +6,5 @@ sources: database: "{{ 'oppiaserver' if target.name == 'prod' else 'oppiatestserver' }}" schema: "{{ 'analytics_261927573' if target.name == 'prod' else 'analytics_264617348' }}" tables: - - name: web_events_log - description: "Raw untransformed event streaming records from the web client application." + - name: events_* + description: "Date-sharded GA4/Firebase event export tables (events_YYYYMMDD); query via wildcard with _TABLE_SUFFIX." diff --git a/models/staging/web/stg_web_analytics__events.sql b/models/staging/web/stg_web_analytics__events.sql index 3123154..0313dbb 100644 --- a/models/staging/web/stg_web_analytics__events.sql +++ b/models/staging/web/stg_web_analytics__events.sql @@ -4,21 +4,41 @@ -- Note: Detailed column descriptions and data quality assertions are managed inside the corresponding schema.yml file. WITH source_data AS ( - SELECT * FROM {{ source('raw_web_server', 'web_events_log') }} + -- events_* wildcard scans all daily shards; _TABLE_SUFFIX exposes each shard's date suffix + SELECT * FROM {{ source('raw_web_server', 'events_*') }} + WHERE _TABLE_SUFFIX BETWEEN FORMAT_DATE('%Y%m%d', DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY)) + AND FORMAT_DATE('%Y%m%d', CURRENT_DATE()) ), final_events AS ( SELECT user_id, - lesson_id, - progress_percent, - updated_at + user_pseudo_id, + event_name, + event_date, + TIMESTAMP_MICROS(event_timestamp) AS event_timestamp, + device.category AS device_category, + device.operating_system AS operating_system, + geo.country AS country, + traffic_source.source AS traffic_source, + traffic_source.medium AS traffic_medium, + -- GA4 stores custom event attributes as key/value pairs rather than fixed columns + (SELECT value.string_value FROM UNNEST(event_params) WHERE key = 'page_location') AS page_location, + (SELECT value.int_value FROM UNNEST(event_params) WHERE key = 'ga_session_id') AS ga_session_id FROM source_data ) SELECT user_id, - lesson_id, - progress_percent, - updated_at + user_pseudo_id, + event_name, + event_date, + event_timestamp, + device_category, + operating_system, + country, + traffic_source, + traffic_medium, + page_location, + ga_session_id FROM final_events From fbf11743feceb2e04f2e76886b7ca62a436855b2 Mon Sep 17 00:00:00 2001 From: Sean Lip Date: Mon, 17 Aug 2026 13:48:51 +0800 Subject: [PATCH 12/12] Address comments --- .github/workflows/pr_validation.yml | 3 +++ macros/generate_surrogate_key.sql | 3 +++ models/staging/web/stg_web_analytics__events.sql | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_validation.yml b/.github/workflows/pr_validation.yml index adee48b..8671505 100644 --- a/.github/workflows/pr_validation.yml +++ b/.github/workflows/pr_validation.yml @@ -51,6 +51,9 @@ jobs: location: US EOF + - name: Install dbt Packages (if any) + run: dbt deps --profiles-dir . + - name: Build and Test Models # dbt build runs each model then tests it immediately, so tests never target an unbuilt table run: dbt build --profiles-dir . diff --git a/macros/generate_surrogate_key.sql b/macros/generate_surrogate_key.sql index 599ff6f..7c80411 100644 --- a/macros/generate_surrogate_key.sql +++ b/macros/generate_surrogate_key.sql @@ -4,6 +4,9 @@ COMPATIBILITY: Google Cloud BigQuery (Standard SQL) USAGE: {{ generate_surrogate_key(['user_id', 'lesson_id']) }} AS assignment_sk -#} +{%- if not field_list -%} + {{ exceptions.raise_compiler_error("generate_surrogate_key() requires a non-empty field_list; CONCAT() with no arguments is invalid in BigQuery.") }} +{%- endif -%} {%- set field_expressions = [] -%} {%- for field in field_list -%} diff --git a/models/staging/web/stg_web_analytics__events.sql b/models/staging/web/stg_web_analytics__events.sql index 0313dbb..89be64a 100644 --- a/models/staging/web/stg_web_analytics__events.sql +++ b/models/staging/web/stg_web_analytics__events.sql @@ -23,8 +23,8 @@ final_events AS ( traffic_source.source AS traffic_source, traffic_source.medium AS traffic_medium, -- GA4 stores custom event attributes as key/value pairs rather than fixed columns - (SELECT value.string_value FROM UNNEST(event_params) WHERE key = 'page_location') AS page_location, - (SELECT value.int_value FROM UNNEST(event_params) WHERE key = 'ga_session_id') AS ga_session_id + (SELECT value.string_value FROM UNNEST(event_params) WHERE key = 'page_location' LIMIT 1) AS page_location, + (SELECT value.int_value FROM UNNEST(event_params) WHERE key = 'ga_session_id' LIMIT 1) AS ga_session_id FROM source_data )