diff --git a/.user.yml b/.user.yml new file mode 100644 index 0000000..b59c4ca --- /dev/null +++ b/.user.yml @@ -0,0 +1 @@ +id: 65fed7dc-cfb1-40ec-af82-fb56180cafce diff --git a/AI_ASSIST.md b/AI_ASSIST.md index 76f6a37..ea7fe3d 100644 --- a/AI_ASSIST.md +++ b/AI_ASSIST.md @@ -4,23 +4,29 @@ Document one place you used an LLM during this assignment. ## The problem - - -TODO - +While building the stg_trips staging model, i had an SQL syntax issue because i placed the "WHERE" clause before the "FROM" clause. ## The prompt - -TODO - +Im building a dbt staging model and my SQL query is failing. what is the problem here? +my query was : +SELECT + pickup_datetime, + pickup_location_id, + fare_amount, + tip_amount, + trip_distance, + + {{ safe_divide('tip_amount', 'fare_amount') }} as tip_pct + +where pickup_location_id is not null +FROM {{ source('nyc_taxi', 'raw_trips') }} + and fare_amount >= 0 ## The response -TODO +the llm explained that SQL clauses follow a fixed order. The FROM clause must come before the WHERE clause because SQL needs to know the source table before applying filters. ## Reflection @@ -29,8 +35,7 @@ TODO 'ratio' to 'tip_pct' to match the assignment schema." --> TODO - ---- +i changed the query order so that "FROM" comes immediately after the SELECT columns, followed by the WHERE filters. > Remember: never paste real connection strings, passwords, or PII into an LLM. > The NYC TLC dataset is public so sample rows are safe here, but practise the habit. diff --git a/docs/Lineage.png b/docs/Lineage.png new file mode 100644 index 0000000..8ca3cc6 Binary files /dev/null and b/docs/Lineage.png differ diff --git a/macros/safe_divide.sql b/macros/safe_divide.sql index e30c8fa..edbbe46 100644 --- a/macros/safe_divide.sql +++ b/macros/safe_divide.sql @@ -3,8 +3,9 @@ -- Use for tip_pct = tip_amount / fare_amount and similar ratio columns. {% macro safe_divide(numerator, denominator) %} - -- TODO: implement the macro body. - -- Use NULLIF(denominator, 0) to avoid division-by-zero errors. - -- Return only the SQL expression (no SELECT, no semicolon). - NULL + + {{ numerator }} / nullif({{ denominator }}, 0) + {% endmacro %} + + diff --git a/models/marts/_fct_daily_borough_stats.yml b/models/marts/_fct_daily_borough_stats.yml index 15bba90..d02fcdf 100644 --- a/models/marts/_fct_daily_borough_stats.yml +++ b/models/marts/_fct_daily_borough_stats.yml @@ -3,22 +3,36 @@ version: 2 models: - name: fct_daily_borough_stats description: > - TODO: state the grain (one row per ___), the source lineage - (built from ___ and ___), and at least one known caveat - (e.g. rows dropped in staging, any WARN-severity tests). + Daily borough-level taxi statistics with a grain of one row per + (pickup_borough, pickup_date). + Built from stg_trips and stg_zones. + Trips with missing or invalid pickup locations are removed during staging + or excluded by the INNER JOIN. + The avg_tip_pct warning test may return some rows where + tips are higher than the fare amount. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - pickup_borough + - pickup_date # TODO: Task 5 -- add the compound uniqueness test on the mart's primary # key (pickup_borough, pickup_date). You need the dbt_utils package for # this: declare it in packages.yml and run `dbt deps` first. columns: - name: pickup_borough - description: "TODO: explain what this column contains and where it comes from" + description: "The TLC borough where trips were picked up, sourced from stg_zones.borough" + tests: + - not_null - name: pickup_date - description: "TODO: explain units and how it is derived" + description: "The calendar date of the trip pickup, derived from pickup_datetime." + tests: + - not_null - name: trip_count - description: "TODO: explain what is counted (unit: number of trips)" + description: "The number of taxi trips picked up in the borough on that dat." - name: total_fare - description: "TODO: explain units (USD) and what fare_amount represents" + description: "The total fare amount collected in US dollars, calculated as the sum of fare_amount" - name: avg_tip_pct - description: "TODO: explain the ratio (tip_amount / fare_amount)" + description: "The average tipping ratio calculated as tip_amount divided by fare_amount." - name: avg_trip_distance - description: "TODO: explain units (miles, from TLC source data)" + description: "The average trip distance in miles based on the TLC trip_distance field" diff --git a/models/marts/fct_daily_borough_stats.sql b/models/marts/fct_daily_borough_stats.sql index bf47070..7aa8977 100644 --- a/models/marts/fct_daily_borough_stats.sql +++ b/models/marts/fct_daily_borough_stats.sql @@ -2,6 +2,7 @@ -- Grain: one row per (pickup_borough, pickup_date). -- Used to answer: trip volume, revenue, tipping behaviour, and distance profile -- per borough per day for January 2024. +{{ config(materialized='table') }} WITH trips AS ( SELECT * @@ -26,13 +27,27 @@ SELECT -- total_fare NUMERIC - sum(fare_amount) -- avg_tip_pct NUMERIC - avg(tip_pct) -- avg_trip_distance NUMERIC - avg(trip_distance) - NULL AS pickup_borough, - NULL AS pickup_date, - NULL AS trip_count, - NULL AS total_fare, - NULL AS avg_tip_pct, - NULL AS avg_trip_distance + z.borough AS pickup_borough, + + t.pickup_datetime::date AS pickup_date, + + count(*) AS trip_count, + + sum(t.fare_amount) AS total_fare, + + avg(t.tip_pct) AS avg_tip_pct, + + avg(t.trip_distance) AS avg_trip_distance + FROM trips t -- TODO: add JOIN to zones here +INNER JOIN zones z + + ON t.pickup_location_id = z.location_id -- TODO: add GROUP BY here +GROUP BY + + z.borough, + + t.pickup_datetime::date diff --git a/models/staging/_sources.yml b/models/staging/_sources.yml index 4bbee9c..288cf98 100644 --- a/models/staging/_sources.yml +++ b/models/staging/_sources.yml @@ -5,6 +5,6 @@ sources: schema: nyc_taxi # TODO: confirm this matches the schema where raw_trips and raw_zones live tables: - name: raw_trips - description: "TODO: one sentence on what this table contains and its grain" + description: "Raw NYC green Taxi trip records, with one row representing one individual taxi trip." - name: raw_zones - description: "TODO: one sentence on what this table contains" + description: "Raw TLC taxi zone lookup table containing location IDs and borough/zone information." diff --git a/models/staging/_stg_trips.yml b/models/staging/_stg_trips.yml index 14829a8..31ad815 100644 --- a/models/staging/_stg_trips.yml +++ b/models/staging/_stg_trips.yml @@ -2,19 +2,23 @@ version: 2 models: - name: stg_trips - description: "TODO: state the grain (one row per ___) and what source this reads from" + description: "One row per taxi trip from the raw_trips source after basic cleaning and filtering." columns: - name: pickup_datetime - description: "TODO" + description: "Timestamp when the trip started" + tests: + - not_null # TODO: Task 5 -- add not_null tests on every column used as a join or # group-by key. See the chapter's dbt Tests section for the syntax. - name: pickup_location_id - description: "TODO" + description: "TLC location ID where the trip was picked up." + tests: + - not_null - name: fare_amount - description: "TODO" + description: "Fare charged for the trip in US dollars" - name: tip_amount - description: "TODO" + description: "Tip paid by the passenger in US dollars" - name: trip_distance - description: "TODO" + description: "Distance travelled during the trip in miles" - name: tip_pct - description: "TODO" + description: "Tip as a percentage of the fare amount" diff --git a/models/staging/_stg_zones.yml b/models/staging/_stg_zones.yml index c4b785d..8e625cf 100644 --- a/models/staging/_stg_zones.yml +++ b/models/staging/_stg_zones.yml @@ -2,11 +2,15 @@ version: 2 models: - name: stg_zones - description: "TODO: state the grain and what source this reads from" + description: "One row per TLC taxi zone from the raw_zones source" columns: - name: location_id - description: "TODO" + description: "Unique identifier for each TLC taxi zone." + tests: + - not_null + - unique + # TODO: Task 5 -- this column is the join key. Which two generic tests # guarantee a clean one-to-many join from stg_trips? - name: borough - description: "TODO" + description: "Borough that the taxi zone belongs to." diff --git a/models/staging/stg_trips.sql b/models/staging/stg_trips.sql index b5f7b81..200da18 100644 --- a/models/staging/stg_trips.sql +++ b/models/staging/stg_trips.sql @@ -1,13 +1,18 @@ -- Staging model: one row per NYC green taxi trip (January 2024). -- Renames source columns, adds derived columns, and filters bad rows. -- Downstream: fct_daily_borough_stats joins this to stg_zones. +{{ config(materialized='view') }} SELECT - -- TODO: select the columns you need for the mart: - -- pickup_datetime, pickup_location_id, fare_amount, tip_amount, trip_distance - -- - -- TODO: add tip_pct using {{ safe_divide('tip_amount', 'fare_amount') }} - -- - -- TODO: filter out rows where pickup_location_id IS NULL or fare_amount < 0 + pickup_datetime, + pickup_location_id, + fare_amount, + tip_amount, + trip_distance, + + {{ safe_divide('tip_amount', 'fare_amount') }} as tip_pct FROM {{ source('nyc_taxi', 'raw_trips') }} + +where pickup_location_id is not null + and fare_amount >= 0 diff --git a/models/staging/stg_zones.sql b/models/staging/stg_zones.sql index cc34d23..9d59a0f 100644 --- a/models/staging/stg_zones.sql +++ b/models/staging/stg_zones.sql @@ -1,7 +1,8 @@ -- Staging model: one row per TLC zone (265 zones). -- Exposes location_id and borough for use as a lookup in the mart. +{{ config(materialized='view') }} SELECT - -- TODO: select location_id and borough from {{ source('nyc_taxi', 'raw_zones') }} - + location_id, + borough FROM {{ source('nyc_taxi', 'raw_zones') }} diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..e397fd7 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,5 @@ +packages: + - name: dbt_utils + package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 226ae69cdfbc9367e2aa2c472b01f99dbce11de0 diff --git a/packages.yml b/packages.yml index bbb2357..ed35da3 100644 --- a/packages.yml +++ b/packages.yml @@ -2,4 +2,6 @@ # `dbt deps` to install it. You need it for the compound uniqueness test # on the mart. See https://hub.getdbt.com/dbt-labs/dbt_utils/latest/ # for the package block syntax. -packages: [] +packages: + - package: dbt-labs/dbt_utils + version: 1.3.0 diff --git a/reports/answers.md b/reports/answers.md index 8fb8cf2..22e2710 100644 --- a/reports/answers.md +++ b/reports/answers.md @@ -1,6 +1,6 @@ # Business Question Answers -Queries run against `dev_.fct_daily_borough_stats`. +Queries run against `dev_baraah.fct_daily_borough_stats`. ## Q1: Highest total `total_fare` across the whole loaded dataset @@ -9,10 +9,20 @@ Queries run against `dev_.fct_daily_borough_stats`. ```sql -- TODO: query fct_daily_borough_stats grouped by pickup_borough, sum total_fare, order DESC ``` - -**Result:** TODO - -**Interpretation:** TODO (one sentence) +SELECT + pickup_borough, + SUM(total_fare) AS total_revenue +FROM dev_baraah.fct_daily_borough_stats +GROUP BY pickup_borough +ORDER BY total_revenue DESC +LIMIT 1; +**Result:** + pickup_borough | total_revenue +----------------+--------------- + Manhattan | 493955.62 +(1 row) + +**Interpretation:** The borough with the highest total fare revenue across the loaded dataset was Manhattan $493955.62. --- @@ -23,10 +33,20 @@ Queries run against `dev_.fct_daily_borough_stats`. ```sql -- TODO: query fct_daily_borough_stats grouped by pickup_date, sum trip_count, order DESC LIMIT 1 ``` - -**Result:** TODO - -**Interpretation:** TODO (one sentence) +SELECT + pickup_date, + SUM(trip_count) AS total_trips +FROM dev_baraah.fct_daily_borough_stats +GROUP BY pickup_date +ORDER BY total_trips DESC +LIMIT 1; +**Result:** + pickup_date | total_trips +-------------+------------- + 2024-01-17 | 2221 +(1 row) + +**Interpretation:** The busiest day in the dataset was 2024-01-17 with 2221 trips across all boroughs. --- @@ -37,10 +57,25 @@ Queries run against `dev_.fct_daily_borough_stats`. ```sql -- TODO: query fct_daily_borough_stats order by avg_tip_pct DESC LIMIT 5 ``` - -**Result:** TODO - -**Interpretation:** TODO — note whether any avg_tip_pct > 1 rows appear and what causes them +SELECT + pickup_borough, + pickup_date, + avg_tip_pct +FROM dev_baraah.fct_daily_borough_stats +ORDER BY avg_tip_pct DESC +LIMIT 5; + +**Result:** + pickup_borough | pickup_date | avg_tip_pct +----------------+-------------+-------------------- + Unknown | 2024-01-30 | 2.500125 + Unknown | 2024-01-07 | 1.3396296296296295 + Unknown | 2024-01-11 | 1.1044956140350877 + Unknown | 2024-01-16 | 1 + Unknown | 2024-01-18 | 0.611111111111111 +(5 rows) + +**Interpretation:** The highest average tip percentage occurred in Unknown on 2024-01-30 . --- @@ -51,7 +86,18 @@ Queries run against `dev_.fct_daily_borough_stats`. ```sql -- TODO: use percentile_cont(0.5) WITHIN GROUP (ORDER BY trip_count) filtered by borough ``` +SELECT + pickup_borough, + percentile_cont(0.5) WITHIN GROUP (ORDER BY trip_count) AS median_trip_count +FROM dev_baraah.fct_daily_borough_stats +WHERE pickup_borough IN ('Manhattan', 'Brooklyn') +GROUP BY pickup_borough; **Result:** TODO + pickup_borough | median_trip_count +----------------+------------------- + Brooklyn | 248 + Manhattan | 1169.5 +(2 rows) -**Interpretation:** TODO (one sentence on the ratio) +**Interpretation:** Manhattan had a higher median daily trip count than Brooklyn. diff --git a/test-output.txt b/test-output.txt new file mode 100644 index 0000000..a5e039c --- /dev/null +++ b/test-output.txt @@ -0,0 +1 @@ +bash: test.sh: No such file or directory diff --git a/tests/assert_avg_tip_pct_within_bounds.sql b/tests/assert_avg_tip_pct_within_bounds.sql index b0fab32..9e91833 100644 --- a/tests/assert_avg_tip_pct_within_bounds.sql +++ b/tests/assert_avg_tip_pct_within_bounds.sql @@ -12,8 +12,15 @@ -- unique_combination primary-key tests, which you want to stay at ERROR. -- -- The test passes (no WARN) when zero rows are returned; any returned rows are flagged. +{{ config(severity='warn') }} -- TODO: write the SELECT here. -- Query {{ ref('fct_daily_borough_stats') }} and return rows where avg_tip_pct > 1. -SELECT NULL AS pickup_borough, NULL AS pickup_date, NULL AS avg_tip_pct -WHERE FALSE -- TODO: replace with the real query +SELECT + pickup_borough, + pickup_date, + avg_tip_pct + +FROM {{ ref('fct_daily_borough_stats') }} + +WHERE avg_tip_pct > 1