diff --git a/.user.yml b/.user.yml new file mode 100644 index 0000000..863f9e2 --- /dev/null +++ b/.user.yml @@ -0,0 +1 @@ +id: c406d767-d5a6-4bff-af8d-712598f1f966 diff --git a/AI_ASSIST.md b/AI_ASSIST.md index 76f6a37..4994187 100644 --- a/AI_ASSIST.md +++ b/AI_ASSIST.md @@ -4,31 +4,33 @@ Document one place you used an LLM during this assignment. ## The problem - - -TODO +when i run dbt run i got an error in my models/stg_trips.sql that was originally my sql error then a database problem ## The prompt - - -TODO +why am i getting this error and how to solve +13:00:05 +13:00:05 Completed with 1 error, 0 partial successes, and 0 warnings: +13:00:05 +13:00:05 Failure in model stg_trips (models\staging\stg_trips.sql) +13:00:05 Database Error in model stg_trips (models\staging\stg_trips.sql) +must be owner of view stg_trips +compiled code at target\run\nyc_taxi_borough_daily\models\staging\stg_trips.sql +13:00:05 +13:00:05 compiled code at target\compiled\nyc_taxi_borough_daily\models\staging\stg_trips.sql +13:00:05 +13:00:05 Done. PASS=1 WARN=0 ERROR=1 SKIP=1 NO-OP=0 TOTAL=3 ## The response - - -TODO +This is a permissions/ownership conflict, not a bug in your SQL. +How to fix it — a few options depending on your access level: +Option 1: Drop the existing view yourself (if you have drop privileges) +Connect to the database (via psql, DBeaver, pgAdmin, etc.) and run: ## Reflection - - -TODO +i dropped my previous view of stg_strips and run dbt then it connected and worked. The error occured as there was an already existing table --- diff --git a/README.md b/README.md index a18f257..958d3d4 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,10 @@ cp profiles.yml.example profiles.yml 2. Export the connection env vars (values are in the class vault; ask your mentor if you are missing them): ```bash -export PG_HOST=... -export PG_USER=... -export PG_PASSWORD=... -export PG_DBNAME=postgres +export PG_HOST=hyf-data-pg.postgres.database.azure.com +export PG_USER=.... +export PG_PASSWORD=..... +export PG_DBNAME=...... ``` 3. Verify: `dbt debug` must end with `All checks passed!`. diff --git a/docs/lineage.png b/docs/lineage.png new file mode 100644 index 0000000..7b0debe Binary files /dev/null and b/docs/lineage.png differ diff --git a/macros/safe_divide.sql b/macros/safe_divide.sql index e30c8fa..709b239 100644 --- a/macros/safe_divide.sql +++ b/macros/safe_divide.sql @@ -2,9 +2,12 @@ -- Returns numerator / denominator, or NULL when denominator is 0 or NULL. -- Use for tip_pct = tip_amount / fare_amount and similar ratio columns. + + + {% macro safe_divide(numerator, denominator) %} - -- TODO: implement the macro body. - -- Use NULLIF(denominator, 0) to avoid division-by-zero errors. - -- Return only the SQL expression (no SELECT, no semicolon). - NULL + {{ numerator }} / NULLIF({{ denominator }}, 0) + {#- NULLIF(x, 0) returns NULL when the denominator is 0, and Postgres propagates + NULL through division, so the whole expression becomes NULL instead of raising + a division-by-zero error. Standard dbt pattern for any ratio column. -#} {% endmacro %} diff --git a/models/marts/_fct_daily_borough_stats.yml b/models/marts/_fct_daily_borough_stats.yml index 15bba90..919a765 100644 --- a/models/marts/_fct_daily_borough_stats.yml +++ b/models/marts/_fct_daily_borough_stats.yml @@ -2,23 +2,29 @@ version: 2 models: - name: fct_daily_borough_stats + description: > - TODO: state the grain (one row per ___), the source lineage - (built from ___ and ___), and at least one known caveat - (e.g. rows dropped in staging, any WARN-severity tests). - # TODO: Task 5 -- add the compound uniqueness test on the mart's primary - # key (pickup_borough, pickup_date). You need the dbt_utils package for - # this: declare it in packages.yml and run `dbt deps` first. + + one row per pickup_borough and pickup_date,the source lineage(built from stg_trips + and stg_zones), and atleast one known caveat(we drop rows in staging where pickup location + id is null)) + test: + dbt_utils.unique_combination_of_columns: + combination_of_columns: + - pickup_borough + - pickup_date columns: - name: pickup_borough - description: "TODO: explain what this column contains and where it comes from" + description: groups trips by the pickup_borough, which is derived from the pickup_location_id and the stg_zones table + - name: pickup_date - description: "TODO: explain units and how it is derived" + description: groups trips by the pickup_date, which is derived from the pickup_datetime in stg_trips + - name: trip_count - description: "TODO: explain what is counted (unit: number of trips)" + description: counts the number of trips for each pickup_borough and pickup_date combination - name: total_fare - description: "TODO: explain units (USD) and what fare_amount represents" + description: sums the fare_amount for each pickup_borough and pickup_date combination - name: avg_tip_pct - description: "TODO: explain the ratio (tip_amount / fare_amount)" + description: calculates the average tip percentage for each pickup_borough and pickup_date combination - name: avg_trip_distance - description: "TODO: explain units (miles, from TLC source data)" + description: calculates the average trip distance for each pickup_borough and pickup_date combination percentage diff --git a/models/marts/fct_daily_borough_stats.sql b/models/marts/fct_daily_borough_stats.sql index bf47070..1413b68 100644 --- a/models/marts/fct_daily_borough_stats.sql +++ b/models/marts/fct_daily_borough_stats.sql @@ -14,25 +14,15 @@ zones AS ( ) SELECT - -- TODO: join trips to zones on pickup_location_id = location_id. - -- Use an INNER JOIN: a few trips have a pickup_location_id with no matching - -- zone (e.g. 999). INNER JOIN drops those so pickup_borough is never NULL and - -- can serve as part of the mart's primary key (your not_null test needs this). - -- TODO: aggregate to grain (pickup_borough, pickup_date) - -- Required output columns: - -- pickup_borough TEXT - z.borough - -- pickup_date DATE - pickup_datetime::date - -- trip_count BIGINT - count(*) - -- total_fare NUMERIC - sum(fare_amount) - -- avg_tip_pct NUMERIC - avg(tip_pct) - -- avg_trip_distance NUMERIC - avg(trip_distance) - NULL AS pickup_borough, - NULL AS pickup_date, - NULL AS trip_count, - NULL AS total_fare, - NULL AS avg_tip_pct, - NULL AS avg_trip_distance + z.borough AS pickup_borough, + t.pickup_datetime::date AS pickup_date, + COUNT(*) AS trip_count, + SUM(t.fare_amount) AS total_fare, + AVG({{ safe_divide('t.tip_amount', 't.fare_amount') }}) AS avg_tip_pct, + AVG(t.trip_distance) AS avg_trip_distance FROM trips t --- TODO: add JOIN to zones here --- TODO: add GROUP BY here +INNER JOIN zones z + ON t.pickup_location_id = z.location_id + +GROUP BY z.borough, t.pickup_datetime::date diff --git a/models/staging/_sources.yml b/models/staging/_sources.yml index 4bbee9c..829ad71 100644 --- a/models/staging/_sources.yml +++ b/models/staging/_sources.yml @@ -2,9 +2,9 @@ version: 2 sources: - name: nyc_taxi - schema: nyc_taxi # TODO: confirm this matches the schema where raw_trips and raw_zones live + schema: nyc_taxi tables: - name: raw_trips - description: "TODO: one sentence on what this table contains and its grain" + description: One trip per row with detailed information - name: raw_zones - description: "TODO: one sentence on what this table contains" + description: a table for taxi zones and their boroughs diff --git a/models/staging/_stg_trips.yml b/models/staging/_stg_trips.yml index 14829a8..5bc5c14 100644 --- a/models/staging/_stg_trips.yml +++ b/models/staging/_stg_trips.yml @@ -2,19 +2,26 @@ version: 2 models: - name: stg_trips - description: "TODO: state the grain (one row per ___) and what source this reads from" + description: cleaned green taxi trips,one row per trip columns: - name: pickup_datetime - description: "TODO" - # TODO: Task 5 -- add not_null tests on every column used as a join or - # group-by key. See the chapter's dbt Tests section for the syntax. + description: When trip started. This is used to group trips by day in the mart + tests: + - not_null + - name: pickup_location_id - description: "TODO" + description: TLC location ID of the pickup location. This is used to join to the zones table in the mart + tests: + - not_null - name: fare_amount - description: "TODO" + description: Total fare amount for the trip + - name: tip_amount - description: "TODO" + description: Tip amount for the trip + - name: trip_distance - description: "TODO" + description: Distance of the trip + - name: tip_pct - description: "TODO" + description: Percentage of the fare that was tipped + \ No newline at end of file diff --git a/models/staging/_stg_zones.yml b/models/staging/_stg_zones.yml index c4b785d..e3b947f 100644 --- a/models/staging/_stg_zones.yml +++ b/models/staging/_stg_zones.yml @@ -2,11 +2,17 @@ version: 2 models: - name: stg_zones - description: "TODO: state the grain and what source this reads from" + description: one row per taxi zone from the columns: - name: location_id - description: "TODO" - # TODO: Task 5 -- this column is the join key. Which two generic tests - # guarantee a clean one-to-many join from stg_trips? + description: TLC location ID of the pickup location. This is used to join to the trips table in the mart + tests: + - not_null + - unique + - name: borough - description: "TODO" + description: The borough of NYC that the taxi zone is in + tests: + - not_null + + \ No newline at end of file diff --git a/models/staging/stg_trips.sql b/models/staging/stg_trips.sql index b5f7b81..e0cb38c 100644 --- a/models/staging/stg_trips.sql +++ b/models/staging/stg_trips.sql @@ -2,12 +2,20 @@ -- Renames source columns, adds derived columns, and filters bad rows. -- Downstream: fct_daily_borough_stats joins this to stg_zones. -SELECT - -- TODO: select the columns you need for the mart: - -- pickup_datetime, pickup_location_id, fare_amount, tip_amount, trip_distance - -- - -- TODO: add tip_pct using {{ safe_divide('tip_amount', 'fare_amount') }} - -- - -- TODO: filter out rows where pickup_location_id IS NULL or fare_amount < 0 + +SELECT + pickup_datetime, + pickup_location_id, + fare_amount, + tip_amount, + trip_distance, + {{ safe_divide('tip_amount', 'fare_amount') }} AS tip_pct FROM {{ source('nyc_taxi', 'raw_trips') }} +WHERE + pickup_location_id IS NOT NULL + AND fare_amount >= 0 + + + + diff --git a/models/staging/stg_zones.sql b/models/staging/stg_zones.sql index cc34d23..e6b910f 100644 --- a/models/staging/stg_zones.sql +++ b/models/staging/stg_zones.sql @@ -1,7 +1,10 @@ -- Staging model: one row per TLC zone (265 zones). -- Exposes location_id and borough for use as a lookup in the mart. + + SELECT - -- TODO: select location_id and borough from {{ source('nyc_taxi', 'raw_zones') }} +location_id, + borough FROM {{ source('nyc_taxi', 'raw_zones') }} diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..cf58e99 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,5 @@ +packages: + - name: dbt_utils + package: dbt-labs/dbt_utils + version: 1.4.1 +sha1_hash: 8b27037b26f3f630c6661194d2470e720c49f6ee diff --git a/packages.yml b/packages.yml index bbb2357..97e3e38 100644 --- a/packages.yml +++ b/packages.yml @@ -2,4 +2,7 @@ # `dbt deps` to install it. You need it for the compound uniqueness test # on the mart. See https://hub.getdbt.com/dbt-labs/dbt_utils/latest/ # for the package block syntax. -packages: [] +packages: + - package: dbt-labs/dbt_utils + version: 1.4.1 + diff --git a/reports/answers.md b/reports/answers.md index 8fb8cf2..5d8ecef 100644 --- a/reports/answers.md +++ b/reports/answers.md @@ -10,9 +10,26 @@ Queries run against `dev_.fct_daily_borough_stats`. -- TODO: query fct_daily_borough_stats grouped by pickup_borough, sum total_fare, order DESC ``` -**Result:** TODO - -**Interpretation:** TODO (one sentence) +select + pickup_borough, + sum(total_fare) as sum_total_fare +from dev_hannahwn.fct_daily_borough_stats +group by pickup_borough +order by sum_total_fare desc; + +**Result:** +pickup_borough|sum_total_fare | +--------------+------------------+ +Manhattan | 493557.47| +Queens | 273361.44| +Brooklyn | 165283.83| +Bronx | 20699.92| +Unknown |2167.4000000000005| +NaN | 1644.51| +EWR | 309.77| +Staten Island |228.09999999999997| + +**Interpretation:** Manhattan has the hghest total fare across the loaded dataset --- @@ -23,10 +40,20 @@ Queries run against `dev_.fct_daily_borough_stats`. ```sql -- TODO: query fct_daily_borough_stats grouped by pickup_date, sum trip_count, order DESC LIMIT 1 ``` +SELECT pickup_date, +sum(trip_count) AS sum_trip_count +FROM dev_hannahwn.fct_daily_borough_stats +GROUP BY pickup_date +ORDER BY sum_trip_count desc +LIMIT 1; -**Result:** TODO -**Interpretation:** TODO (one sentence) +**Result:** +pickup_date|sum_trip_count| +-----------+--------------+ + 2024-01-17| 2225| + +**Interpretation:** ON 2024/01/17 they did 2225 trips --- @@ -38,7 +65,21 @@ Queries run against `dev_.fct_daily_borough_stats`. -- TODO: query fct_daily_borough_stats order by avg_tip_pct DESC LIMIT 5 ``` -**Result:** TODO +SELECT +pickup_date,pickup_borough, +avg_tip_pct +FROM dev_hannahwn.fct_daily_borough_stats +ORDER BY avg_tip_pct DESC +LIMIT 5; + +**Result:** +pickup_date|pickup_borough|avg_tip_pct | +-----------+--------------+------------------+ + 2024-01-30|Unknown | 2.500125| + 2024-01-07|Unknown |1.3396296296296295| + 2024-01-11|Unknown |1.1044956140350877| + 2024-01-16|Unknown | 1.0| + 2024-01-18|Unknown | 0.611111111111111| **Interpretation:** TODO — note whether any avg_tip_pct > 1 rows appear and what causes them @@ -52,6 +93,12 @@ Queries run against `dev_.fct_daily_borough_stats`. -- TODO: use percentile_cont(0.5) WITHIN GROUP (ORDER BY trip_count) filtered by borough ``` +SELECT pickup_borough, +trip_count +FROM dev_hannahwn.fct_daily_borough_stats +GROUP BY +ORDER BY trip_count + **Result:** TODO **Interpretation:** TODO (one sentence on the ratio) diff --git a/tests/assert_avg_tip_pct_within_bounds.sql b/tests/assert_avg_tip_pct_within_bounds.sql index b0fab32..1c0b5a4 100644 --- a/tests/assert_avg_tip_pct_within_bounds.sql +++ b/tests/assert_avg_tip_pct_within_bounds.sql @@ -3,17 +3,14 @@ -- which is unusual and almost always indicates a small-sample bucket (e.g. the -- Unknown borough) where a few high-tip outliers dominate the average. -- --- Set this test to WARN severity by adding an inline config at the top of this --- file (below these comments): {{ config(severity='warn') }} --- That keeps a few expected Unknown-borough rows from blocking `dbt build`, while --- still surfacing them for your reports/answers.md write-up (the rubric requires --- documenting this finding). Do NOT set a project-level test severity in --- dbt_project.yml: that would also downgrade your not_null and --- unique_combination primary-key tests, which you want to stay at ERROR. --- --- The test passes (no WARN) when zero rows are returned; any returned rows are flagged. --- TODO: write the SELECT here. --- Query {{ ref('fct_daily_borough_stats') }} and return rows where avg_tip_pct > 1. -SELECT NULL AS pickup_borough, NULL AS pickup_date, NULL AS avg_tip_pct -WHERE FALSE -- TODO: replace with the real query +{{ config(severity='warn') }} + +SELECT + pickup_borough, + pickup_date, + avg_tip_pct +FROM {{ ref('fct_daily_borough_stats') }} +WHERE avg_tip_pct > 1 + +