From d6866158a3a33c922aefda8dafa62291d493ce24 Mon Sep 17 00:00:00 2001 From: mohanab-db <121183444+mohanab-db@users.noreply.github.com> Date: Mon, 16 Sep 2024 17:09:50 -0500 Subject: [PATCH] Additional job metrics added --- .../Jobs-PrPr/Jobs System Tables Dashboard.lvdash.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/System-Tables-Demo/Jobs-PrPr/Jobs System Tables Dashboard.lvdash.json b/System-Tables-Demo/Jobs-PrPr/Jobs System Tables Dashboard.lvdash.json index 05a9259..07e1c03 100644 --- a/System-Tables-Demo/Jobs-PrPr/Jobs System Tables Dashboard.lvdash.json +++ b/System-Tables-Demo/Jobs-PrPr/Jobs System Tables Dashboard.lvdash.json @@ -1 +1 @@ -{"datasets":[{"name":"7801704c","displayName":"finished_jobs","query":"SELECT * FROM system.lakeflow.job_run_timeline WHERE result_state IS NOT NULL;"},{"name":"c980521d","displayName":"most_retry_jobs_30_days","query":"with job_run_timeline_with_cost as (\n SELECT\n t1.*,\n t2.job_id,\n t2.run_id,\n t1.identity_metadata.run_as as run_as,\n t2.result_state,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1\n INNER JOIN system.lakeflow.job_run_timeline t2\n ON\n t1.workspace_id=t2.workspace_id\n AND t1.usage_metadata.job_id = t2.job_id\n AND t1.usage_metadata.job_run_id = t2.run_id\n AND t1.usage_start_time >= date_trunc(\"Hour\", t2.period_start_time)\n AND t1.usage_start_time < date_trunc(\"Hour\", t2.period_end_time) + INTERVAL 1 HOUR\n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%' AND \n t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAYS\n),\ncumulative_run_status_cost as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n run_as,\n result_state,\n usage_end_time,\n SUM(list_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_cost\n FROM job_run_timeline_with_cost\n ORDER BY workspace_id, job_id, run_id, usage_end_time\n),\ncost_per_status as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n run_as,\n result_state,\n usage_end_time,\n cumulative_cost - COALESCE(LAG(cumulative_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time), 0) AS result_state_cost\n FROM cumulative_run_status_cost\n WHERE result_state IS NOT NULL\n ORDER BY workspace_id, job_id, run_id, usage_end_time),\ncost_per_unsuccesful_status_agg as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n first(run_as, TRUE) as run_as,\n SUM(result_state_cost) as list_cost\n FROM cost_per_status\n WHERE\n result_state != \"SUCCEEDED\"\n GROUP BY ALL\n),\nrepaired_runs as (\n SELECT \n workspace_id, job_id, run_id, COUNT(*) as cnt\n FROM system.lakeflow.job_run_timeline\n WHERE result_state IS NOT NULL\n GROUP BY ALL\n HAVING cnt > 1\n),\nsuccessful_repairs as (\n SELECT t1.workspace_id, t1.job_id, t1.run_id, MAX(t1.period_end_time) as period_end_time\n FROM system.lakeflow.job_run_timeline t1\n JOIN repaired_runs t2\n ON t1.workspace_id=t2.workspace_id AND t1.job_id=t2.job_id AND t1.run_id=t2.run_id\n WHERE t1.result_state=\"SUCCEEDED\"\n GROUP BY ALL\n),\ncombined_repairs as (\n SELECT\n t1.*,\n t2.period_end_time, \n t1.cnt as repairs\n FROM repaired_runs t1\n LEFT JOIN successful_repairs t2 USING (workspace_id, job_id, run_id)\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT\n last(t3.name) as name,\n t1.workspace_id,\n t1.job_id,\n t1.run_id,\n first(t4.run_as, TRUE) as run_as,\n first(t1.repairs) - 1 as repairs,\n first(t4.list_cost) as repair_list_cost,\n CASE WHEN t1.period_end_time IS NOT NULL THEN CAST(t1.period_end_time - MIN(t2.period_end_time) as LONG) ELSE NULL END AS repair_time_seconds\nFROM combined_repairs t1\n JOIN system.lakeflow.job_run_timeline t2 USING (workspace_id, job_id, run_id)\n LEFT JOIN most_recent_jobs t3 USING (workspace_id, job_id)\n LEFT JOIN cost_per_unsuccesful_status_agg t4 USING (workspace_id, job_id, run_id)\nWHERE\n t2.result_state IS NOT NULL\nGROUP BY t1.workspace_id, t1.job_id, t1.run_id, t1.period_end_time\nORDER BY repairs DESC"},{"name":"709a2764","displayName":"wow_growth","query":"with jobs_usage_with_list_cost as (\n SELECT\n t1.*,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%' \n),\nweekly_jobs_list_cost as (\n select\n date_sub(to_date(usage_start_time), dayofweek(to_date(usage_start_time))) as week_start,\n SUM(list_cost) as list_cost,\n COUNT(distinct to_date(usage_start_time)) as days\n from\n jobs_usage_with_list_cost t1 \n group by all\n having days = 7\n order by 1 asc, 2 asc\n),\nlist_cost_growth as (\n select\n week_start,\n list_cost,\n 100 * (list_cost - lag(list_cost, 1) over (order by week_start)) / lag(list_cost, 1) over (order by week_start) as weekly_growth\n from weekly_jobs_list_cost\n order by week_start asc\n)\nselect\n week_start,\n weekly_growth,\n avg(weekly_growth) over (order by week_start ROWS BETWEEN 12 preceding AND CURRENT ROW) as 3_months_growth_moving_avg,\n list_cost\nfrom list_cost_growth"},{"name":"11db8b54","displayName":"jobs_stale_datasets","query":"with producers as (\n SELECT \n workspace_id, entity_id as job_id, target_table_full_name, created_by\n FROM\n system.access.table_lineage\n WHERE\n entity_type=\"JOB\" AND \n target_table_full_name IS NOT NULL AND\n event_date > CURRENT_DATE() - INTERVAL 30 DAY\n GROUP BY ALL\n),\nconsumers as (\n SELECT \n workspace_id, source_table_full_name\n FROM\n system.access.table_lineage\n WHERE\n source_table_full_name IS NOT NULL AND event_date > CURRENT_DATE() - INTERVAL 30 DAY\n GROUP BY ALL\n),\njobs_with_datasets as (\n SELECT \n workspace_id, entity_id as job_id, collect_set(target_table_full_name) as populated_sets\n FROM\n system.access.table_lineage\n WHERE\n entity_type=\"JOB\" AND \n target_table_full_name IS NOT NULL\n GROUP BY ALL\n),\njobs_without_consumption as (\n SELECT \n t1.*\n FROM producers t1\n LEFT OUTER JOIN consumers t2\n ON t1.target_table_full_name = t2.source_table_full_name AND t1.workspace_id=t2.workspace_id\n),\njob_run_timeline_with_cost as (\n SELECT\n t1.*,\n t1.usage_metadata.job_id as job_id,\n t1.usage_metadata.job_run_id as run_id,\n t1.identity_metadata.run_as as run_as,\n t1.usage_quantity * list_prices.pricing.default AS list_cost\n FROM system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices \n ON\n t1.cloud = list_prices.cloud AND\n t1.sku_name = list_prices.sku_name AND\n t1.usage_start_time >= list_prices.price_start_time AND\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is NULL)\n WHERE\n t1.sku_name LIKE '%JOBS%' AND\n t1.usage_metadata.job_id IS NOT NULL AND\n t1.usage_metadata.job_run_id IS NOT NULL\n),\nagg_cost as (\n SELECT \n workspace_id,\n job_id,\n FIRST(run_as, TRUE) as run_as,\n SUM(list_cost) as list_cost\n FROM job_run_timeline_with_cost\n WHERE\n usage_start_time >= CURRENT_DATE() - INTERVAL 30 DAYS \n GROUP BY ALL\n ORDER BY list_cost DESC\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT\n t1.workspace_id,\n t5.name, \n t1.job_id,\n collect_set(target_table_full_name) as not_used_datasets,\n array_except(first(t3.populated_sets), collect_set(target_table_full_name)) as used_datasets,\n first(t4.run_as) as run_as,\n date(MAX(t2.period_end_time)) as last_seen_job_date,\n COUNT(DISTINCT t2.run_id) as runs,\n first(t4.list_cost) as list_cost_30d\nFROM \n jobs_without_consumption t1 \n INNER JOIN jobs_with_datasets t3 USING (workspace_id, job_id)\n LEFT JOIN system.lakeflow.job_run_timeline t2 USING (workspace_id, job_id)\n LEFT JOIN agg_cost t4 USING (workspace_id, job_id)\n LEFT JOIN most_recent_jobs t5 USING (workspace_id, job_id)\nGROUP BY ALL\nHAVING runs > 0 -- Exclude old Lineage entries for which we don't have jobs data\nORDER BY list_cost_30d DESC, runs DESC"},{"name":"1ed3a5b7","displayName":"highest_change_jobs_7_14_days","query":"with job_run_timeline_with_cost as (\n SELECT\n t1.*,\n t1.usage_metadata.job_id as job_id,\n t1.identity_metadata.run_as as run_as,\n t1.usage_quantity * list_prices.pricing.default AS list_cost\n FROM system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices \n ON\n t1.cloud = list_prices.cloud AND\n t1.sku_name = list_prices.sku_name AND\n t1.usage_start_time >= list_prices.price_start_time AND\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is NULL)\n WHERE\n t1.sku_name LIKE '%JOBS%' AND\n t1.usage_metadata.job_id IS NOT NULL AND\n t1.usage_metadata.job_run_id IS NOT NULL AND\n t1.usage_date >= CURRENT_DATE() - INTERVAL 14 DAY\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT\n t2.name\n ,t1.workspace_id\n ,t1.job_id\n ,t1.sku_name\n ,t1.run_as\n ,Last7DaySpend\n ,Last14DaySpend\n ,last7DaySpend - last14DaySpend as Last7DayGrowth\n ,try_divide( (last7DaySpend - last14DaySpend) , last14DaySpend) * 100 AS Last7DayGrowthPct\nFROM\n (\n SELECT\n workspace_id,\n job_id,\n run_as,\n sku_name,\n SUM(list_cost) AS spend\n ,SUM(CASE WHEN usage_end_time BETWEEN date_add(current_date(), -8) AND date_add(current_date(), -1) THEN list_cost ELSE 0 END) AS Last7DaySpend\n ,SUM(CASE WHEN usage_end_time BETWEEN date_add(current_date(), -15) AND date_add(current_date(), -8) THEN list_cost ELSE 0 END) AS Last14DaySpend\n FROM job_run_timeline_with_cost\n GROUP BY ALL\n ) t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nORDER BY \n Last7DayGrowth DESC \nLIMIT 100"},{"name":"9e75b6fd","displayName":"cost_per_status_jobs_30_days","query":"with job_run_timeline_with_cost as (\n SELECT\n t1.cloud,\n t1.workspace_id,\n t2.job_id,\n t2.run_id,\n t2.result_state,\n t1.usage_start_time,\n t1.usage_end_time,\n t1.sku_name,\n t1.usage_unit,\n t1.usage_quantity,\n t1.usage_quantity * list_prices.pricing.default AS list_cost\n FROM system.billing.usage t1\n INNER JOIN system.lakeflow.job_run_timeline t2\n ON\n t1.workspace_id=t2.workspace_id\n AND t1.usage_metadata.job_id = t2.job_id\n AND t1.usage_metadata.job_run_id = t2.run_id\n AND t1.usage_start_time >= date_trunc(\"Hour\", t2.period_start_time)\n AND t1.usage_start_time < date_trunc(\"Hour\", t2.period_end_time) + INTERVAL 1 HOUR\n INNER JOIN system.billing.list_prices list_prices \n ON\n t1.cloud = list_prices.cloud AND\n t1.sku_name = list_prices.sku_name AND\n t1.usage_start_time >= list_prices.price_start_time AND\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is NULL)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n),\ncumulative_run_status_cost as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n result_state,\n usage_start_time,\n usage_end_time,\n SUM(list_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_cost\n FROM job_run_timeline_with_cost\n ORDER BY workspace_id, job_id, run_id, usage_end_time\n )\nSELECT\n workspace_id, \n job_id,\n run_id,\n result_state,\n usage_start_time,\n usage_end_time,\n cumulative_cost - COALESCE(LAG(cumulative_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time), 0) AS result_state_cost\nFROM cumulative_run_status_cost\nWHERE result_state IS NOT NULL\nORDER BY workspace_id, job_id, run_id, usage_end_time"},{"name":"2d7e1f43","displayName":"job_run_cost_agg_30_days","query":"with jobs_usage_with_list_cost as (\n SELECT\n t1.*,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%' \n AND usage_date >= CURRENT_DATE() - INTERVAL 30 DAYS \n),\njobs_usage_per_workspace as (\n SELECT\n workspace_id, \n SUM(list_cost) as list_cost\n FROM jobs_usage_with_list_cost\n GROUP BY ALL\n)\nSELECT \n t1.workspace_id,\n COUNT(DISTINCT t1.job_id) as num_jobs,\n COUNT(DISTINCT t1.run_id) as num_runs,\n first(t2.list_cost) as list_cost\nFROM system.lakeflow.job_run_timeline t1\n LEFT JOIN jobs_usage_per_workspace t2 USING (workspace_id)\nWHERE\n period_start_time >= CURRENT_DATE() - INTERVAL 30 DAYS \nGROUP BY ALL\nORDER BY list_cost DESC\n"},{"name":"04affe8d","displayName":"job_run_cost","query":"SELECT\n t1.*,\n t1.usage_metadata.job_id as job_id,\n t1.usage_metadata.job_run_id as run_id,\n t1.identity_metadata.run_as as run_as,\n t1.usage_quantity * list_prices.pricing.default AS list_cost\nFROM system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices \n ON\n t1.cloud = list_prices.cloud AND\n t1.sku_name = list_prices.sku_name AND\n t1.usage_start_time >= list_prices.price_start_time AND\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is NULL)\nWHERE\n t1.sku_name LIKE '%JOBS%' AND\n t1.usage_metadata.job_id IS NOT NULL AND \n t1.usage_metadata.job_run_id IS NOT NULL"},{"name":"b4c689de","displayName":"most_expensive_jobs_30_days","query":"with list_cost_per_job as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC"},{"name":"f463e18e","displayName":"outlier_job_runs_30_days","query":"-- SPENDING BROKEN DOWN BY JOB\nwith stats_per_job_run as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.job_run_id as run_id,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen,\n MIN(t1.usage_start_time) as first_seen\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_metadata.job_run_id IS NOT NULL\n AND t1.usage_start_time >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY 1,2,3\n), \nstats_per_job AS (\n SELECT\n workspace_id,\n job_id,\n COUNT(run_id) AS runs,\n SUM(list_cost) AS total_list_cost,\n AVG(list_cost) AS avg_list_cost,\n MAX(list_cost) AS max_list_cost,\n percentile(list_cost, 0.9) AS p90\n FROM\n stats_per_job_run\n WHERE\n first_seen > CURRENT_TIMESTAMP() - INTERVAL 30 DAYS\n GROUP BY ALL\n ORDER BY\n total_list_cost DESC\n)\nSELECT\n *,\n (max_list_cost - p90) AS list_cost_deviation_from_p90\nFROM\n stats_per_job\nORDER BY\n list_cost_deviation_from_p90 DESC"},{"name":"0ff3e918","displayName":"30_60_days_spend","query":"with jobs_usage_with_list_cost as (\n SELECT\n t1.*,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n)\nSELECT\n *\n ,try_divide( (last30DaySpend - last30to60DaySpend) , last30to60DaySpend) * 100 AS Last30DayGrowth\nFROM\n (\n SELECT\n SUM(list_cost) AS spend\n ,sku_name\n ,SUM(CASE WHEN usage_end_time BETWEEN date_add(current_date(), -30) AND current_date() THEN list_cost ELSE 0 END) AS Last30DaySpend\n ,SUM(CASE WHEN usage_end_time BETWEEN date_add(current_date(), -60) AND date_add(current_date(), -31) THEN list_cost ELSE 0 END) AS Last30to60DaySpend\n FROM jobs_usage_with_list_cost\n GROUP BY ALL\n )"},{"name":"44c189cb","displayName":"most_expensive_runs_30_days","query":"with list_cost_per_job_run as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.job_run_id as run_id,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_metadata.job_run_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\n\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT \n t1.workspace_id,\n t2.name,\n t1.job_id,\n t1.run_id,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_run t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC"},{"name":"9080da05","displayName":"highest_failure_jobs_30_days","query":"with job_run_timeline_with_cost as (\n SELECT\n t1.*,\n t1.identity_metadata.run_as as run_as,\n t2.job_id,\n t2.run_id,\n t2.result_state,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1\n INNER JOIN system.lakeflow.job_run_timeline t2\n ON\n t1.workspace_id=t2.workspace_id\n AND t1.usage_metadata.job_id = t2.job_id\n AND t1.usage_metadata.job_run_id = t2.run_id\n AND t1.usage_start_time >= date_trunc(\"Hour\", t2.period_start_time)\n AND t1.usage_start_time < date_trunc(\"Hour\", t2.period_end_time) + INTERVAL 1 HOUR\n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%' AND \n t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAYS\n),\ncumulative_run_status_cost as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n run_as,\n result_state,\n usage_end_time,\n SUM(list_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_cost\n FROM job_run_timeline_with_cost\n ORDER BY workspace_id, job_id, run_id, usage_end_time\n),\ncost_per_status as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n run_as,\n result_state,\n usage_end_time,\n cumulative_cost - COALESCE(LAG(cumulative_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time), 0) AS result_state_cost\n FROM cumulative_run_status_cost\n WHERE result_state IS NOT NULL\n ORDER BY workspace_id, job_id, run_id, usage_end_time),\ncost_per_status_agg as (\n SELECT\n workspace_id, \n job_id,\n FIRST(run_as, TRUE) as run_as,\n SUM(result_state_cost) as list_cost\n FROM cost_per_status\n WHERE\n result_state IN ('ERROR', 'FAILED', 'TIMED_OUT')\n GROUP BY ALL\n),\nterminal_statues as (\n SELECT\n workspace_id,\n job_id,\n CASE WHEN result_state IN ('ERROR', 'FAILED', 'TIMED_OUT') THEN 1 ELSE 0 END as is_failure,\n period_end_time as last_seen_date\n FROM system.lakeflow.job_run_timeline\n WHERE \n result_state IS NOT NULL AND\n period_end_time >= CURRENT_DATE() - INTERVAL 30 DAYS\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT\n first(t2.name) as name, \n t1.workspace_id,\n t1.job_id,\n COUNT(*) as runs,\n t3.run_as,\n SUM(is_failure) as failures,\n (1 - COALESCE(try_divide(SUM(is_failure), COUNT(*)), 0)) * 100 as success_ratio,\n first(t3.list_cost) as failure_list_cost,\n MAX(t1.last_seen_date) as last_seen_date\nFROM terminal_statues t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\n LEFT JOIN cost_per_status_agg t3 USING (workspace_id, job_id)\nGROUP BY ALL \nORDER BY failures DESC"},{"name":"a6199734","displayName":"job_cost_savings","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\tand t1.sku_name not ilike '%jobs_serverless%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\njobsrollingmonth as (\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC ),\n\nclusterutil as (SELECT * from \n\tsystem.compute.node_timeline\nWHERE\n\tstart_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\n\njobswithutil as (\nselect jobsrollingmonth.workspace_id, jobsrollingmonth.name,\navg(cpu_user_percent + cpu_system_percent) as `Avg CPU Utilization`, \nmax(cpu_user_percent + cpu_system_percent) as `Peak CPU Utilization`, \n\tavg(cpu_wait_percent) as `Avg CPU Disk Wait`,\n\tmax(cpu_wait_percent) as `Max CPU Disk Wait`,\n\tavg(mem_used_percent) as `Avg Memory Utilization`, \n\tmax(mem_used_percent) as `Max Memory Utilization`\nfrom clusterutil \njoin jobsrollingmonth\non clusterutil.cluster_id = jobsrollingmonth.cluster_id\ngroup by 1, 2),\n\nfinalutil as (\n select jobswithutil.*, sum(jobsrollingmonth.list_cost) as jobcostrolling30\n from jobswithutil\n join jobsrollingmonth\n on jobswithutil.workspace_id = jobsrollingmonth.workspace_id\n AND jobswithutil.name = jobsrollingmonth.name \n group by ALL\n)\n\nselect *, ((jobcostrolling30 * (1 - (`Avg CPU Utilization` / 100))) / 2.0) as maxsavings\nfrom finalutil\norder by maxsavings desc\nlimit 25"},{"name":"387e1015","displayName":"job_low_cpu","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\tand t1.sku_name not ilike '%jobs_serverless%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\njobsrollingmonth as (\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC ),\n\nclusterutil as (SELECT * from \n\tsystem.compute.node_timeline\nWHERE\n\tstart_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\n\njobswithutil as (\nselect jobsrollingmonth.workspace_id, jobsrollingmonth.name,\navg(cpu_user_percent + cpu_system_percent) as `Avg CPU Utilization`, \nmax(cpu_user_percent + cpu_system_percent) as `Peak CPU Utilization`, \n\tavg(cpu_wait_percent) as `Avg CPU Disk Wait`,\n\tmax(cpu_wait_percent) as `Max CPU Disk Wait`,\n\tavg(mem_used_percent) as `Avg Memory Utilization`, \n\tmax(mem_used_percent) as `Max Memory Utilization`\nfrom clusterutil \njoin jobsrollingmonth\non clusterutil.cluster_id = jobsrollingmonth.cluster_id\ngroup by 1, 2)\n\nselect *\nfrom jobswithutil\norder by `Avg CPU Utilization` asc\nlimit 10"},{"name":"b4382d02","displayName":"job_high_mem","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\tand t1.sku_name not ilike '%jobs_serverless%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\njobsrollingmonth as (\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC ),\n\nclusterutil as (SELECT * from \n\tsystem.compute.node_timeline\nWHERE\n\tstart_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\n\njobswithutil as (\nselect jobsrollingmonth.workspace_id, jobsrollingmonth.name,\navg(cpu_user_percent + cpu_system_percent) as `Avg CPU Utilization`, \nmax(cpu_user_percent + cpu_system_percent) as `Peak CPU Utilization`, \n\tavg(cpu_wait_percent) as `Avg CPU Disk Wait`,\n\tmax(cpu_wait_percent) as `Max CPU Disk Wait`,\n\tavg(mem_used_percent) as `Avg Memory Utilization`, \n\tmax(mem_used_percent) as `Max Memory Utilization`\nfrom clusterutil \njoin jobsrollingmonth\non clusterutil.cluster_id = jobsrollingmonth.cluster_id\ngroup by 1, 2)\n\nselect *\nfrom jobswithutil\norder by `Avg Memory Utilization` desc\nlimit 10"},{"name":"bb2993fe","displayName":"spend_by_tag","query":"with tagraw as (\n select explode(custom_tags) as (llave, val), (usage_quantity * list_prices.pricing.default)\n as list_cost \n from system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\t and t1.sku_name not ilike '%jobs_serverless%'\n and t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY),\n\ntagspend as ( select concat(llave,':',' ', val) as tagbucket, sum(list_cost) as list_cost\nfrom tagraw \ngroup by 1),\n\ncatchall as (select 'catchall' as tagbucket , sum(usage_quantity * list_prices.pricing.default)\nas list_cost \n from system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\t and t1.sku_name not ilike '%jobs_serverless%'\n and t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY\n and cardinality(custom_tags) = 0 )\n\nselect * from tagspend\nunion all\nselect * from catchall\norder by list_cost desc\nlimit 20"},{"name":"b95eb997","displayName":"job_low_dbr","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\tand t1.sku_name not ilike '%jobs_serverless%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\njobsrollingmonth as (\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC ),\n\nclusters as (SELECT * from \n\tsystem.compute.clusters\nWHERE\n\tdelete_time is not null\n)\n\nselect clusters.workspace_id, jobsrollingmonth.name, jobsrollingmonth.job_id,\njobsrollingmonth.run_as, sum(jobsrollingmonth.list_cost), any_value(dbr_Version) from \nclusters join jobsrollingmonth\non clusters.cluster_id = jobsrollingmonth.cluster_id\nwhere left(dbr_version, 1) <> '1'\ngroup by 1, 2, 3, 4\n"},{"name":"d2f17553","displayName":"job_no_scaling","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\t AND t1.sku_name NOT LIKE '%SERVERLESS'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_metadata.cluster_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\njob_runs_enhanced as (\n SELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\n FROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\n GROUP BY ALL\n ORDER BY list_cost DESC\n),\nclusters as (\n SELECT \n *,\n ROW_NUMBER() OVER(PARTITION BY workspace_id, cluster_id ORDER BY change_time DESC) as rn\n FROM \n system.compute.clusters\n WHERE\n cluster_source=\"JOB\"\n QUALIFY rn=1\n)\nSELECT\n clusters.workspace_id,\n jobs.job_id,\n first(jobs.name) as name,\n first(jobs.run_as) as run_as,\n SUM(jobs.list_cost) as list_cost,\n first(worker_count) as worker_count,\n first(min_autoscale_workers) as min_autoscale_workers,\n first(max_autoscale_workers) as max_autoscale_workers\nFROM clusters\n JOIN job_runs_enhanced jobs USING(workspace_id, cluster_id)\nWHERE\n (min_autoscale_workers is NULL OR max_autoscale_workers is NULL)\n and worker_count > 1\nGROUP BY 1,2\nORDER BY list_cost DESC"},{"name":"ed6a4a62","displayName":"jobs_no_tags","query":"with most_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\nrunlast30 as (\n\n select * from system.lakeflow.job_run_timeline\n where period_end_time > CURRENT_DATE - INTERVAL 30 DAY\n)\n\n\nselect most_recent_jobs.workspace_id, name, any_value(most_recent_jobs.job_id), \nany_value(tags), any_value(run_as) from most_recent_jobs\njoin runlast30\non most_recent_jobs.job_id = runlast30.job_id\nwhere cardinality(tags) = 0 \ngroup by all\n"},{"name":"bd7c848b","displayName":"jobs_ap_clusters","query":"with clusters AS (\n SELECT\n *,\n ROW_NUMBER() OVER(PARTITION BY workspace_id, cluster_id ORDER BY change_time DESC) as rn\n FROM system.compute.clusters\n WHERE cluster_source=\"UI\"\n QUALIFY rn=1\n),\njob_tasks_exploded AS (\n SELECT\n workspace_id,\n job_id,\n EXPLODE(compute_ids) as cluster_id\n FROM system.lakeflow.job_task_run_timeline\n WHERE period_start_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\nall_purpose_cluster_jobs AS (\n SELECT\n t1.*,\n t2.cluster_name,\n t2.owned_by,\n t2.dbr_version\n FROM job_tasks_exploded t1 \n INNER JOIN clusters t2 USING (workspace_id, cluster_id)\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT \n t1.workspace_id,\n t1.job_id,\n first(t2.name, TRUE) as name,\n array_size(collect_list(t1.cluster_id)) as task_runs,\n collect_set(\n named_struct(\n 'cluster_id', t1.cluster_id,\n 'cluster_name', t1.cluster_name\n )\n ) as clusters_list\nFROM all_purpose_cluster_jobs t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY t1.workspace_id, t1.job_id \nORDER BY task_runs DESC\nLIMIT 100;"},{"name":"5b55fedd","displayName":"ap_clusters_by_jobs","query":"with clusters AS (\n SELECT\n *,\n ROW_NUMBER() OVER(PARTITION BY workspace_id, cluster_id ORDER BY change_time DESC) as rn\n FROM system.compute.clusters\n WHERE cluster_source=\"UI\"\n QUALIFY rn=1\n),\njob_tasks_exploded AS (\n SELECT\n workspace_id,\n job_id,\n EXPLODE(compute_ids) as cluster_id\n FROM system.lakeflow.job_task_run_timeline\n WHERE period_start_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\nall_purpose_cluster_jobs AS (\n SELECT\n t1.*,\n t2.cluster_name,\n t2.owned_by,\n t2.dbr_version\n FROM job_tasks_exploded t1 \n INNER JOIN clusters t2 USING (workspace_id, cluster_id)\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n-- )\n),\nap_clusters_by_jobs as (\n SELECT \n t1.workspace_id,\n t1.cluster_id,\n first(t1.cluster_name, TRUE) as cluster_name,\n array_size(collect_list(t1.cluster_id)) as task_runs,\n collect_set(\n named_struct(\n 'job_id', t1.job_id,\n 'job_name', t2.name\n )\n ) as jobs_list\n FROM all_purpose_cluster_jobs t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\n GROUP BY t1.workspace_id, t1.cluster_id \n ORDER BY task_runs DESC\n LIMIT 100\n)\nSELECT\n t1.workspace_id,\n t1.usage_metadata.cluster_id,\n first(t2.cluster_name, TRUE) as cluster_name,\n first(t2.task_runs) as task_runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(t2.jobs_list) as jobs_list,\n MAX(t1.usage_end_time) as last_seen_date\nFROM system.billing.usage t1 \nINNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\nINNER JOIN ap_clusters_by_jobs t2 ON t1.workspace_id=t2.workspace_id AND t1.usage_metadata.cluster_id=t2.cluster_id\nWHERE usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \nGROUP BY ALL\nORDER BY list_cost DESC"}],"pages":[{"name":"400c64ef","displayName":"New Page","layout":[{"widget":{"name":"cdd8a91f","queries":[{"name":"main_query","query":{"datasetName":"c980521d","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"name","expression":"`name`"},{"name":"repair_list_cost","expression":"`repair_list_cost`"},{"name":"repair_time_seconds","expression":"`repair_time_seconds`"},{"name":"repairs","expression":"`repairs`"},{"name":"run_as","expression":"`run_as`"},{"name":"run_id","expression":"`run_id`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":4,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"repairs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":5,"title":"repairs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"repairs"},{"fieldName":"repair_time_seconds","numberFormat":"0,","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":6,"title":"repair_time","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"description":"Repair time in seconds","displayName":"repair_time_seconds"},{"fieldName":"repair_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":7,"title":"repair_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"repair_list_cost"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Most Retried Runs Last 30 Days","showDescription":true,"description":"- NULL repair time means there was no successful repair (yet)\n- We include all the jobs, but the run_as/repair_$list are calculated only for these running on serverless or job clusters."}}},"position":{"x":0,"y":135,"width":6,"height":10}},{"widget":{"name":"e2d11219","queries":[{"name":"main_query","query":{"datasetName":"0ff3e918","fields":[{"name":"sum(Last30to60DaySpend)","expression":"SUM(`Last30to60DaySpend`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"counter","encodings":{"value":{"fieldName":"sum(Last30to60DaySpend)","displayName":"Sum of Last30to60DaySpend"}},"frame":{"showTitle":true,"title":"30-60 Day $list Spend","showDescription":false}}},"position":{"x":0,"y":11,"width":1,"height":2}},{"widget":{"name":"e0921e07","queries":[{"name":"main_query","query":{"datasetName":"9080da05","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"name","expression":"`name`"},{"name":"job_id","expression":"`job_id`"},{"name":"run_as","expression":"`run_as`"},{"name":"runs","expression":"`runs`"},{"name":"failures","expression":"`failures`"},{"name":"success_ratio","expression":"`success_ratio`"},{"name":"failure_list_cost","expression":"`failure_list_cost`"},{"name":"last_seen_date","expression":"`last_seen_date`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":4,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"runs"},{"fieldName":"failures","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":5,"title":"failures","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"failures"},{"fieldName":"success_ratio","numberFormat":"0.0%","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"float","displayAs":"number","visible":true,"order":6,"title":"success_ratio","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"cellFormat":{"default":{"foregroundColor":null},"rules":[{"if":{"column":"success_ratio","fn":"<","literal":"95"},"value":{"foregroundColor":"#E92828"}}]},"displayName":"success_ratio"},{"fieldName":"failure_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":7,"title":"failure_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"failure_list_cost"},{"fieldName":"last_seen_date","dateTimeFormat":"YYYY-MM-DD","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"datetime","displayAs":"datetime","visible":true,"order":8,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"last_seen_date"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":true,"frame":{"showTitle":true,"title":"Highest Failure Jobs Last 30 Days","showDescription":true,"description":"We include all the jobs, but the run_as/failure_$list are calculated only for these running on serverless or job clusters."}}},"position":{"x":0,"y":126,"width":6,"height":9}},{"widget":{"name":"9172b6ab","queries":[{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1115189aa1b763efc94c0c34_period_end_time","query":{"datasetName":"7801704c","fields":[{"name":"period_end_time","expression":"`period_end_time`"},{"name":"period_end_time_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_usage_end_time","query":{"datasetName":"04affe8d","fields":[{"name":"usage_end_time","expression":"`usage_end_time`"},{"name":"usage_end_time_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11251513a8c486a1267b2242_usage_end_time","query":{"datasetName":"9e75b6fd","fields":[{"name":"usage_end_time","expression":"`usage_end_time`"},{"name":"usage_end_time_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-date-range-picker","encodings":{"fields":[{"fieldName":"period_end_time","displayName":"period_end_time","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1115189aa1b763efc94c0c34_period_end_time"},{"fieldName":"usage_end_time","displayName":"usage_end_time","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_usage_end_time"},{"fieldName":"usage_end_time","displayName":"usage_end_time","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11251513a8c486a1267b2242_usage_end_time"}]},"frame":{"showTitle":true,"title":"Date Range"}}},"position":{"x":0,"y":7,"width":3,"height":2}},{"widget":{"name":"d78e8e9d","queries":[{"name":"main_query","query":{"datasetName":"b4c689de","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"name","expression":"`name`"},{"name":"job_id","expression":"`job_id`"},{"name":"run_as","expression":"`run_as`"},{"name":"runs","expression":"`runs`"},{"name":"list_cost","expression":"`list_cost`"},{"name":"last_seen_date","expression":"`last_seen_date`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":4,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"runs"},{"fieldName":"list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":5,"title":"list_cost","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost"},{"fieldName":"last_seen_date","dateTimeFormat":"YYYY-MM-DD","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"datetime","displayAs":"datetime","visible":true,"order":6,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"last_seen_date"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":true,"frame":{"showTitle":true,"title":"Most Expensive Jobs Last 30 Days"}}},"position":{"x":0,"y":37,"width":6,"height":9}},{"widget":{"name":"ea22bf1d","queries":[{"name":"main_query","query":{"datasetName":"9e75b6fd","fields":[{"name":"daily(usage_end_time)","expression":"DATE_TRUNC(\"DAY\", `usage_end_time`)"},{"name":"result_state","expression":"`result_state`"},{"name":"sum(result_state_cost)","expression":"SUM(`result_state_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"daily(usage_end_time)","scale":{"type":"temporal"},"displayName":"date"},"y":{"fieldName":"sum(result_state_cost)","scale":{"type":"quantitative"},"axis":{"title":"% of $list by run status"},"displayName":"% of $list by run status"},"color":{"fieldName":"result_state","scale":{"type":"categorical","mappings":[{"value":"SUCCEEDED","color":"#00A972"},{"value":"FAILED","color":"#FCA4A1"},{"value":"ERROR","color":"#FF3621"},{"value":"SKIPPED","color":"#8BCAE7"},{"value":"TIMED_OUT","color":"#919191"}]},"displayName":"result_state"}},"frame":{"showTitle":true,"title":"Daily $list by Run Status Distribution","showDescription":false,"description":""},"mark":{"layout":"percent-stack"}}},"position":{"x":3,"y":31,"width":3,"height":6}},{"widget":{"name":"92ffb565","queries":[{"name":"main_query","query":{"datasetName":"04affe8d","fields":[{"name":"sku_name","expression":"`sku_name`"},{"name":"daily(usage_end_time)","expression":"DATE_TRUNC(\"DAY\", `usage_end_time`)"},{"name":"sum(list_cost)","expression":"SUM(`list_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"daily(usage_end_time)","scale":{"type":"temporal"},"displayName":"date"},"y":{"fieldName":"sum(list_cost)","scale":{"type":"quantitative"},"displayName":"$DBU","axis":{"title":"$list"}},"color":{"fieldName":"sku_name","scale":{"type":"categorical"},"displayName":"sku_name"}},"frame":{"showTitle":true,"title":"Daily $list by SKU","showDescription":true,"description":""}}},"position":{"x":3,"y":25,"width":3,"height":6}},{"widget":{"name":"36a4fd54","queries":[{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766215f0a7ef0292360d6895_job_id","query":{"datasetName":"9080da05","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee1607664184e811fbf8ae29c6f9b_job_id","query":{"datasetName":"7801704c","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766618a3988d3d1574121eb9_job_id","query":{"datasetName":"b4c689de","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee16076681a21be8d1a87b12f7a7e_job_id","query":{"datasetName":"9e75b6fd","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766a1a36821c722538cd0b97_job_id","query":{"datasetName":"44c189cb","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766c1a238c191c3fcd462999_job_id","query":{"datasetName":"c980521d","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160767514eaaa492671b6de5b24_job_id","query":{"datasetName":"1ed3a5b7","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee1607677147a8912a7296ca363d6_job_id","query":{"datasetName":"04affe8d","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee6b68a0d1199bfe2a4a773bce474_job_id","query":{"datasetName":"11db8b54","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eef8fde24c1d6aaed907d5e57845fa_job_id","query":{"datasetName":"f463e18e","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee1607664184e811fbf8ae29c6f9b_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee1607677147a8912a7296ca363d6_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766618a3988d3d1574121eb9_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160767514eaaa492671b6de5b24_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766a1a36821c722538cd0b97_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee16076681a21be8d1a87b12f7a7e_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766215f0a7ef0292360d6895_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766c1a238c191c3fcd462999_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee6b68a0d1199bfe2a4a773bce474_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eef8fde24c1d6aaed907d5e57845fa_job_id"}]},"frame":{"showTitle":true,"showDescription":false,"title":"Job Id"}}},"position":{"x":4,"y":7,"width":1,"height":2}},{"widget":{"name":"da14f0b4","queries":[{"name":"main_query","query":{"datasetName":"44c189cb","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"last_seen_date","expression":"`last_seen_date`"},{"name":"list_cost","expression":"`list_cost`"},{"name":"name","expression":"`name`"},{"name":"run_as","expression":"`run_as`"},{"name":"run_id","expression":"`run_id`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100001,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100002,"title":"job_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100003,"title":"run_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100004,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100005,"title":"$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"description":"","displayName":"list_cost"},{"fieldName":"last_seen_date","dateTimeFormat":"YYYY-MM-DD HH:mm:ss.SSS","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"datetime","displayAs":"datetime","visible":true,"order":100006,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"last_seen_date"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":true,"frame":{"showTitle":true,"title":"Most Expensive Runs Last 30 Days"}}},"position":{"x":0,"y":46,"width":6,"height":9}},{"widget":{"name":"45299df0","queries":[{"name":"main_query","query":{"datasetName":"04affe8d","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"daily(usage_end_time)","expression":"DATE_TRUNC(\"DAY\", `usage_end_time`)"},{"name":"sum(list_cost)","expression":"SUM(`list_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"daily(usage_end_time)","scale":{"type":"temporal"},"displayName":"date"},"y":{"fieldName":"sum(list_cost)","scale":{"type":"quantitative"},"displayName":"$DBU","axis":{"title":"$list"}},"color":{"fieldName":"workspace_id","scale":{"type":"categorical"},"displayName":"workspace_id"}},"frame":{"showTitle":true,"title":"Daily $list by Workspace"}}},"position":{"x":3,"y":9,"width":3,"height":6}},{"widget":{"name":"ab292823","queries":[{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1115189aa1b763efc94c0c34_workspace_id","query":{"datasetName":"7801704c","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1128131fb9e01cac845917fd_workspace_id","query":{"datasetName":"2d7e1f43","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112c1eec8f52d7f62b5747c9_workspace_id","query":{"datasetName":"b4c689de","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1122109b80eafe6a8ef44c4b_workspace_id","query":{"datasetName":"1ed3a5b7","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1135149380718c60d46a1635_workspace_id","query":{"datasetName":"44c189cb","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11251513a8c486a1267b2242_workspace_id","query":{"datasetName":"9e75b6fd","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1138158697aa8f1c17fd4792_workspace_id","query":{"datasetName":"9080da05","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11181b3285cbe1695cbcaf4b_workspace_id","query":{"datasetName":"c980521d","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112f17c7886968566cca9912_workspace_id","query":{"datasetName":"f463e18e","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d111e1120a7fdf215a6d76f6c_workspace_id","query":{"datasetName":"11db8b54","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_workspace_id","query":{"datasetName":"04affe8d","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1115189aa1b763efc94c0c34_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1128131fb9e01cac845917fd_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112c1eec8f52d7f62b5747c9_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1122109b80eafe6a8ef44c4b_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1135149380718c60d46a1635_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11251513a8c486a1267b2242_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1138158697aa8f1c17fd4792_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11181b3285cbe1695cbcaf4b_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112f17c7886968566cca9912_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d111e1120a7fdf215a6d76f6c_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_workspace_id"}]},"frame":{"showTitle":true,"title":"Workspace Id"}}},"position":{"x":3,"y":7,"width":1,"height":2}},{"widget":{"name":"b3985e25","textbox_spec":"## Tutorial\n\n---\n\n### How to get a job by ID?\n1. Copy the job_id\n2. Navigate to the correct workspace\n3. Paste the job ID in a top search toolbar\n4. Press enter\n\n### How to get a job run by ID?\n1. Follow the *How to get a job by ID?* section\n2. Copy run ID\n3. Open *any run* on the job runs page\n4. Replace the run ID in the url after `/runs/` part\n 1. *https://sample-workspace.databricks.com/jobs/123456789/runs/**111213141516***"},"position":{"x":0,"y":162,"width":6,"height":6}},{"widget":{"name":"4414b30e","queries":[{"name":"main_query","query":{"datasetName":"0ff3e918","fields":[{"name":"sum(Last30DaySpend)","expression":"SUM(`Last30DaySpend`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"counter","encodings":{"value":{"fieldName":"sum(Last30DaySpend)","displayName":"Sum of Last30DaySpend"}},"frame":{"showTitle":true,"title":"30 Day $list Spend","showDescription":false,"description":""}}},"position":{"x":0,"y":9,"width":1,"height":2}},{"widget":{"name":"11b002f8","queries":[{"name":"main_query","query":{"datasetName":"0ff3e918","fields":[{"name":"Last30DayGrowth","expression":"`Last30DayGrowth`"}],"disaggregated":true}}],"spec":{"version":2,"widgetType":"counter","encodings":{"value":{"fieldName":"Last30DayGrowth","displayName":"Last30DayGrowth"}},"frame":{"showTitle":true,"title":"30 Day Growth %","showDescription":false}}},"position":{"x":0,"y":13,"width":1,"height":2}},{"widget":{"name":"9bd0b48d","queries":[{"name":"main_query","query":{"datasetName":"709a2764","fields":[{"name":"3_months_growth_moving_avg","expression":"`3_months_growth_moving_avg`"},{"name":"week_start","expression":"`week_start`"},{"name":"weekly_growth","expression":"`weekly_growth`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"combo","encodings":{"x":{"fieldName":"week_start","scale":{"type":"temporal"},"displayName":"week_start"},"y":{"primary":{"fields":[{"fieldName":"weekly_growth","displayName":"weekly %change"}]},"secondary":{"fields":[{"fieldName":"3_months_growth_moving_avg","displayName":"3_months_growth_moving_avg"}]},"scale":{"type":"quantitative"}}},"frame":{"showTitle":true,"title":"WoW $list %change"}}},"position":{"x":0,"y":25,"width":3,"height":6}},{"widget":{"name":"34dfeb31","queries":[{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1138158697aa8f1c17fd4792_run_as","query":{"datasetName":"9080da05","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112c1eec8f52d7f62b5747c9_run_as","query":{"datasetName":"b4c689de","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11181b3285cbe1695cbcaf4b_run_as","query":{"datasetName":"c980521d","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1135149380718c60d46a1635_run_as","query":{"datasetName":"44c189cb","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1122109b80eafe6a8ef44c4b_run_as","query":{"datasetName":"1ed3a5b7","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d111e1120a7fdf215a6d76f6c_run_as","query":{"datasetName":"11db8b54","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_run_as","query":{"datasetName":"04affe8d","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1138158697aa8f1c17fd4792_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112c1eec8f52d7f62b5747c9_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11181b3285cbe1695cbcaf4b_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1135149380718c60d46a1635_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1122109b80eafe6a8ef44c4b_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d111e1120a7fdf215a6d76f6c_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_run_as"}]},"frame":{"showTitle":true,"title":"Run as "}}},"position":{"x":5,"y":7,"width":1,"height":2}},{"widget":{"name":"abd1e7a4","textbox_spec":"## Jobs System Tables Dashboard v0.2\n\n--- \n\nThis dashboard operates on a list cost (**$list**). The list cost is calculated as `number of DBUs * list price per DBU at that moment in time`. Cost calculation only includes serverless jobs and job clusters. Interactive/SQL clusters are excluded. Account-specific rates and discounts are excluded.\n\nWe will be evolving it over time by adding more example queries and visualizations. The latest version will always be available in the GitHub link below.\n\n#### Documentation \n- Jobs System Tables (Private Preview) - Checkout the PDF attachment in the email titled **Welcome to Jobs System Tables Private Preview**, sent to the person who unrolled this account to the PrPr\n- [Latest Dashboard version GitHub](https://github.com/databricks/tmm/blob/main/System-Tables-Demo/Jobs-PrPr/Jobs%20System%20Tables%20Dashboard.lvdash.json)\n\n#### Requirements\n- For the full experience, this dashboard requires select rights on **system.usage**, **system.lakeflow** and **system.access.table_lineage** System Tables\n - [How to Enable System Table Schema](https://docs.databricks.com/en/administration-guide/system-tables/index.html#enable-system-table-schemas)\n - [Grant access to system tables](https://docs.databricks.com/en/administration-guide/system-tables/index.html#grant-access-to-system-tables)\n\n#### Debugging\nFor detailed debugging instructions (how to find a job/run by ID) check out the guidance at the bottom of this dashboard."},"position":{"x":0,"y":0,"width":6,"height":7}},{"widget":{"name":"2aae3808","queries":[{"name":"main_query","query":{"datasetName":"f463e18e","fields":[{"name":"avg_list_cost","expression":"`avg_list_cost`"},{"name":"job_id","expression":"`job_id`"},{"name":"list_cost_deviation_from_p90","expression":"`list_cost_deviation_from_p90`"},{"name":"max_list_cost","expression":"`max_list_cost`"},{"name":"p90","expression":"`p90`"},{"name":"runs","expression":"`runs`"},{"name":"total_list_cost","expression":"`total_list_cost`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100001,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100002,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"runs"},{"fieldName":"total_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100003,"title":"total_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"total_list_cost"},{"fieldName":"avg_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100004,"title":"avg_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"avg_list_cost"},{"fieldName":"max_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100005,"title":"max_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"description":"","displayName":"max_list_cost"},{"fieldName":"p90","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"float","displayAs":"number","visible":true,"order":100006,"title":"p90","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"p90"},{"fieldName":"list_cost_deviation_from_p90","numberFormat":"0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"float","displayAs":"number","visible":true,"order":100007,"title":"$list_deviation_from_p90","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost_deviation_from_p90"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"showDescription":true,"title":"Jobs Outliers Last 30 Days","description":"Analyse the job's $list deviation across multiple runs. "}}},"position":{"x":0,"y":145,"width":6,"height":8}},{"widget":{"name":"9a043919","queries":[{"name":"main_query","query":{"datasetName":"7801704c","fields":[{"name":"countdistinct(run_id)","expression":"COUNT(DISTINCT `run_id`)"},{"name":"daily(period_end_time)","expression":"DATE_TRUNC(\"DAY\", `period_end_time`)"},{"name":"result_state","expression":"`result_state`"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"daily(period_end_time)","scale":{"type":"temporal"},"displayName":"date"},"y":{"fieldName":"countdistinct(run_id)","scale":{"type":"quantitative"},"axis":{"title":"% of jobs by run status"},"displayName":"% of jobs by run status"},"color":{"fieldName":"result_state","scale":{"type":"categorical","mappings":[{"value":"ERROR","color":"#FF3621"},{"value":"SUCCEEDED","color":"#00A972"},{"value":"SKIPPED","color":"#8BCAE7"},{"value":"FAILED","color":"#FCA4A1"},{"value":"TIMED_OUT","color":"#919191"}]},"displayName":"result_state"}},"frame":{"showTitle":true,"title":"Daily Run Status Distribution","showDescription":true,"description":""},"mark":{"layout":"percent-stack"}}},"position":{"x":0,"y":31,"width":3,"height":6}},{"widget":{"name":"b9cb4778","queries":[{"name":"main_query","query":{"datasetName":"11db8b54","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"last_seen_job_date","expression":"`last_seen_job_date`"},{"name":"list_cost_30d","expression":"`list_cost_30d`"},{"name":"name","expression":"`name`"},{"name":"not_used_datasets","expression":"`not_used_datasets`"},{"name":"run_as","expression":"`run_as`"},{"name":"runs","expression":"`runs`"},{"name":"used_datasets","expression":"`used_datasets`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":4,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"runs"},{"fieldName":"not_used_datasets","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"complex","displayAs":"json","visible":true,"order":5,"title":"not_used_datasets","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"not_used_datasets"},{"fieldName":"used_datasets","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"complex","displayAs":"json","visible":true,"order":6,"title":"used_datasets","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"used_datasets"},{"fieldName":"last_seen_job_date","dateTimeFormat":"YYYY-MM-DD","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"date","displayAs":"datetime","visible":true,"order":7,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"last_seen_job_date"},{"fieldName":"list_cost_30d","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":8,"title":"$list_30d","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost_30d"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Stale Datasets produced by Jobs","showDescription":true,"description":"Identify datasets without consumers"}}},"position":{"x":0,"y":153,"width":6,"height":9}},{"widget":{"name":"b0965d80","queries":[{"name":"main_query","query":{"datasetName":"1ed3a5b7","fields":[{"name":"Last14DaySpend","expression":"`Last14DaySpend`"},{"name":"Last7DayGrowth","expression":"`Last7DayGrowth`"},{"name":"Last7DayGrowthPct","expression":"`Last7DayGrowthPct`"},{"name":"Last7DaySpend","expression":"`Last7DaySpend`"},{"name":"job_id","expression":"`job_id`"},{"name":"name","expression":"`name`"},{"name":"run_as","expression":"`run_as`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"Last7DaySpend","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":4,"title":"Last7DaySpend","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"Last7DaySpend"},{"fieldName":"Last14DaySpend","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":5,"title":"Last14DaySpend","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"Last14DaySpend"},{"fieldName":"Last7DayGrowth","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":6,"title":"Last7DayGrowth","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"Last7DayGrowth"},{"fieldName":"Last7DayGrowthPct","numberFormat":"0.00%","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":7,"title":"Last7DayGrowthPct","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"cellFormat":{"default":{"foregroundColor":null},"rules":[{"if":{"column":"Last7DayGrowthPct","fn":">","literal":"30"},"value":{"foregroundColor":"#E92828"}}]},"displayName":"Last7DayGrowthPct"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"sku_name","type":"string","displayAs":"string","order":8,"title":"sku_name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":true,"frame":{"showTitle":true,"title":"Highest change in weekly $list (past 2 weeks)"}}},"position":{"x":0,"y":55,"width":6,"height":8}},{"widget":{"name":"b0f7ac9b","queries":[{"name":"main_query","query":{"datasetName":"2d7e1f43","fields":[{"name":"num_jobs","expression":"`num_jobs`"},{"name":"num_runs","expression":"`num_runs`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"num_jobs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100001,"title":"jobs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"num_jobs"},{"fieldName":"num_runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100002,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"num_runs"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"list_cost","type":"decimal","displayAs":"number","order":100003,"title":"list_cost","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Most Expensive Workspaces Last 30 Days ","showDescription":true,"description":"Based on Jobs Clusters & Serverless Jobs Cost"}}},"position":{"x":1,"y":9,"width":2,"height":6}},{"widget":{"name":"65467a29","queries":[{"name":"main_query","query":{"datasetName":"a6199734","fields":[{"name":"name","expression":"`name`"},{"name":"sum(maxsavings)","expression":"SUM(`maxsavings`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"name","axis":{"hideLabels":true,"title":"job name"},"scale":{"type":"categorical","sort":{"by":"y-reversed"}},"displayName":"job name"},"y":{"fieldName":"sum(maxsavings)","scale":{"type":"quantitative"},"displayName":"Max savings $list"},"color":{"fieldName":"name","scale":{"type":"categorical"},"legend":{"hideTitle":false,"position":"right"},"displayName":"name"}},"frame":{"showTitle":true,"title":"Top 25 Jobs by Potential Savings per Month","showDescription":true,"description":"These jobs provide highest ROI for optimization because they tend to be both pricey to run and also not close to fully leveraging harward potential. The calculation is optimistic-- it assumes that if you are at 20% utilization you can get similar performance with 20% of the hardware. Total savings would likely be lower."}}},"position":{"x":0,"y":67,"width":6,"height":9}},{"widget":{"name":"ccac1a39","queries":[{"name":"main_query","query":{"datasetName":"387e1015","fields":[{"name":"name","expression":"`name`"},{"name":"median(Avg CPU Utilization)","expression":"PERCENTILE(`Avg CPU Utilization`, 0.5)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"name","axis":{"hideLabels":true,"title":"job name"},"scale":{"type":"categorical","sort":{"by":"y-reversed"}},"displayName":"job name"},"y":{"fieldName":"median(Avg CPU Utilization)","scale":{"type":"quantitative"},"displayName":"Avg CPU Utilization %"},"color":{"fieldName":"name","scale":{"type":"categorical"},"displayName":"name"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Top 10 Jobs with Lowest Avg CPU Utilization","showDescription":true,"description":"These jobs are not close to fully leveraging the hardware potential - consider reducing cores or removing single-threaded code/ manual repartitions."}}},"position":{"x":0,"y":76,"width":3,"height":7}},{"widget":{"name":"360582c7","queries":[{"name":"main_query","query":{"datasetName":"bb2993fe","fields":[{"name":"sum(list_cost)","expression":"SUM(`list_cost`)"},{"name":"tagbucket","expression":"`tagbucket`"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"pie","encodings":{"angle":{"fieldName":"sum(list_cost)","scale":{"type":"quantitative"},"displayName":"Sum of list_cost"},"color":{"fieldName":"tagbucket","scale":{"type":"categorical"},"displayName":"tagbucket"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Rolling 30 $list by Tag","showDescription":true,"description":""}}},"position":{"x":0,"y":15,"width":6,"height":10}},{"widget":{"name":"748b7fc6","queries":[{"name":"main_query","query":{"datasetName":"b4382d02","fields":[{"name":"name","expression":"`name`"},{"name":"median(Avg Memory Utilization)","expression":"PERCENTILE(`Avg Memory Utilization`, 0.5)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"name","scale":{"type":"categorical","sort":{"by":"y-reversed"}},"axis":{"title":"job name","hideLabels":true},"displayName":"job name"},"y":{"fieldName":"median(Avg Memory Utilization)","scale":{"type":"quantitative"},"axis":{"hideLabels":false},"displayName":"Avg Memory Utilization %"},"color":{"fieldName":"name","scale":{"type":"categorical"},"displayName":"name"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Top 10 Jobs with Highest Avg Memory Utilization","showDescription":true,"description":"These jobs could experience bottlenecks - consider increasing memory allocation. "}}},"position":{"x":3,"y":76,"width":3,"height":7}},{"widget":{"name":"99a10bd4","queries":[{"name":"main_query","query":{"datasetName":"d2f17553","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"name","expression":"`name`"},{"name":"job_id","expression":"`job_id`"},{"name":"run_as","expression":"`run_as`"},{"name":"list_cost","expression":"`list_cost`"},{"name":"worker_count","expression":"`worker_count`"},{"name":"min_autoscale_workers","expression":"`min_autoscale_workers`"},{"name":"max_autoscale_workers","expression":"`max_autoscale_workers`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":4,"title":"list_cost","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost"},{"fieldName":"worker_count","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":5,"title":"worker_count","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"worker_count"},{"fieldName":"min_autoscale_workers","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":6,"title":"min_autoscale_workers","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"min_autoscale_workers"},{"fieldName":"max_autoscale_workers","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":7,"title":"max_autoscale_workers","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"max_autoscale_workers"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Jobs with Fixed Number of Workers Last 30 Days","showDescription":true,"description":"These jobs do not leverage autoscaling, consider adding it to optimise the resources utilisation."}}},"position":{"x":0,"y":83,"width":6,"height":9}},{"widget":{"name":"3af566db","queries":[{"name":"main_query","query":{"datasetName":"b95eb997","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"name","expression":"`name`"},{"name":"job_id","expression":"`job_id`"},{"name":"any_value(dbr_Version)","expression":"`any_value(dbr_Version)`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"any_value(dbr_Version)","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"DBR version","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"any_value(dbr_Version)"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"run_as","type":"string","displayAs":"string","order":4,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"sum(list_cost)","type":"decimal","displayAs":"number","order":5,"title":"sum(list_cost)","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Jobs Running on Outdated DBR Version Last 30 Days","showDescription":true,"description":"These jobs are on old runtimes - consider upgrading to ensure optimal performance and ability to use new features"}}},"position":{"x":0,"y":92,"width":6,"height":8}},{"widget":{"name":"596c7e27","textbox_spec":"## Job Clusters Analysis \n----\nThe below charts and tables enrich job system table data with compute system table data to identify jobs with low CPU utilization, jobs with high memory utilization, jobs w/o scaling or auto-terminate, and jobs on low DBRs. Further it provides a ballpark estimate of jobs with highest savings potential based on current spend and current utilization. \n\nFor the following charts to work you need to leverage:\n- lakeflow system tables\n- compute system tables, include node_timeline Private Preview"},"position":{"x":0,"y":63,"width":6,"height":4}},{"widget":{"name":"0b6f62c1","textbox_spec":"## Jobs Operational Observability "},"position":{"x":0,"y":124,"width":6,"height":2}},{"widget":{"name":"2f042e21","queries":[{"name":"main_query","query":{"datasetName":"ed6a4a62","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"any_value(job_id)","expression":"`any_value(job_id)`"},{"name":"name","expression":"`name`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"any_value(job_id)","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"any_value(job_id)"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"any_value(tags)","type":"complex","displayAs":"json","order":3,"title":"any_value(tags)","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"any_value(run_as)","type":"string","displayAs":"string","order":4,"title":"any_value(run_as)","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"showDescription":true,"title":"Untagged Jobs Last 30 Days","description":"These jobs are not tagged and have been run in last 30 days; consider tagging them for cost attribution"}}},"position":{"x":0,"y":100,"width":6,"height":8}},{"widget":{"name":"5c56abee","queries":[{"name":"main_query","query":{"datasetName":"bd7c848b","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"job_id","expression":"`job_id`"},{"name":"name","expression":"`name`"},{"name":"task_runs","expression":"`task_runs`"},{"name":"clusters_list","expression":"`clusters_list`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100001,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100002,"title":"name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"task_runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100003,"title":"task_runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"task_runs"},{"fieldName":"clusters_list","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"complex","displayAs":"json","visible":true,"order":100004,"title":"clusters_list","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"clusters_list"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Most Frequent Jobs on All Purpose Clusters Last 30 Days","showDescription":true,"description":"High frequency jobs executed on all purpose clusters might prevent them from shutting down."}}},"position":{"x":0,"y":108,"width":6,"height":8}},{"widget":{"name":"2197aeb0","queries":[{"name":"main_query","query":{"datasetName":"5b55fedd","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"cluster_id","expression":"`cluster_id`"},{"name":"cluster_name","expression":"`cluster_name`"},{"name":"task_runs","expression":"`task_runs`"},{"name":"list_cost","expression":"`list_cost`"},{"name":"jobs_list","expression":"`jobs_list`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"cluster_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100001,"title":"cluster_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"cluster_id"},{"fieldName":"cluster_name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100002,"title":"cluster_name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"cluster_name"},{"fieldName":"task_runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100003,"title":"task_runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"task_runs"},{"fieldName":"list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100004,"title":"list_cost","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost"},{"fieldName":"jobs_list","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"complex","displayAs":"json","visible":true,"order":100005,"title":"jobs_list","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"jobs_list"}]},"invisibleColumns":[{"dateTimeFormat":"YYYY-MM-DD HH:mm:ss.SSS","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"last_seen_date","type":"datetime","displayAs":"datetime","order":100006,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":true,"withRowNumber":false,"frame":{"showTitle":true,"title":"All Purpose Clusters with the Highest Job Run Frequency Last 30 Days","showDescription":true,"description":"List of your all purpose clusters most frequently used to run jobs on. "}}},"position":{"x":0,"y":116,"width":6,"height":8}}]}]} \ No newline at end of file +{"datasets":[{"name":"e5c48fa4","displayName":"finished_jobs","query":"SELECT * FROM system.lakeflow.job_run_timeline WHERE result_state IS NOT NULL;"},{"name":"cc789c72","displayName":"most_retry_jobs_30_days","query":"with job_run_timeline_with_cost as (\n SELECT\n t1.*,\n t2.job_id,\n t2.run_id,\n t1.identity_metadata.run_as as run_as,\n t2.result_state,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1\n INNER JOIN system.lakeflow.job_run_timeline t2\n ON\n t1.workspace_id=t2.workspace_id\n AND t1.usage_metadata.job_id = t2.job_id\n AND t1.usage_metadata.job_run_id = t2.run_id\n AND t1.usage_start_time >= date_trunc(\"Hour\", t2.period_start_time)\n AND t1.usage_start_time < date_trunc(\"Hour\", t2.period_end_time) + INTERVAL 1 HOUR\n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%' AND \n t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAYS\n),\ncumulative_run_status_cost as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n run_as,\n result_state,\n usage_end_time,\n SUM(list_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_cost\n FROM job_run_timeline_with_cost\n ORDER BY workspace_id, job_id, run_id, usage_end_time\n),\ncost_per_status as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n run_as,\n result_state,\n usage_end_time,\n cumulative_cost - COALESCE(LAG(cumulative_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time), 0) AS result_state_cost\n FROM cumulative_run_status_cost\n WHERE result_state IS NOT NULL\n ORDER BY workspace_id, job_id, run_id, usage_end_time),\ncost_per_unsuccesful_status_agg as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n first(run_as, TRUE) as run_as,\n SUM(result_state_cost) as list_cost\n FROM cost_per_status\n WHERE\n result_state != \"SUCCEEDED\"\n GROUP BY ALL\n),\nrepaired_runs as (\n SELECT \n workspace_id, job_id, run_id, COUNT(*) as cnt\n FROM system.lakeflow.job_run_timeline\n WHERE result_state IS NOT NULL\n GROUP BY ALL\n HAVING cnt > 1\n),\nsuccessful_repairs as (\n SELECT t1.workspace_id, t1.job_id, t1.run_id, MAX(t1.period_end_time) as period_end_time\n FROM system.lakeflow.job_run_timeline t1\n JOIN repaired_runs t2\n ON t1.workspace_id=t2.workspace_id AND t1.job_id=t2.job_id AND t1.run_id=t2.run_id\n WHERE t1.result_state=\"SUCCEEDED\"\n GROUP BY ALL\n),\ncombined_repairs as (\n SELECT\n t1.*,\n t2.period_end_time, \n t1.cnt as repairs\n FROM repaired_runs t1\n LEFT JOIN successful_repairs t2 USING (workspace_id, job_id, run_id)\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT\n last(t3.name) as name,\n t1.workspace_id,\n t1.job_id,\n t1.run_id,\n first(t4.run_as, TRUE) as run_as,\n first(t1.repairs) - 1 as repairs,\n first(t4.list_cost) as repair_list_cost,\n CASE WHEN t1.period_end_time IS NOT NULL THEN CAST(t1.period_end_time - MIN(t2.period_end_time) as LONG) ELSE NULL END AS repair_time_seconds\nFROM combined_repairs t1\n JOIN system.lakeflow.job_run_timeline t2 USING (workspace_id, job_id, run_id)\n LEFT JOIN most_recent_jobs t3 USING (workspace_id, job_id)\n LEFT JOIN cost_per_unsuccesful_status_agg t4 USING (workspace_id, job_id, run_id)\nWHERE\n t2.result_state IS NOT NULL\nGROUP BY t1.workspace_id, t1.job_id, t1.run_id, t1.period_end_time\nORDER BY repairs DESC"},{"name":"bb7e5c78","displayName":"wow_growth","query":"with jobs_usage_with_list_cost as (\n SELECT\n t1.*,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%' \n),\nweekly_jobs_list_cost as (\n select\n date_sub(to_date(usage_start_time), dayofweek(to_date(usage_start_time))) as week_start,\n SUM(list_cost) as list_cost,\n COUNT(distinct to_date(usage_start_time)) as days\n from\n jobs_usage_with_list_cost t1 \n group by all\n having days = 7\n order by 1 asc, 2 asc\n),\nlist_cost_growth as (\n select\n week_start,\n list_cost,\n 100 * (list_cost - lag(list_cost, 1) over (order by week_start)) / lag(list_cost, 1) over (order by week_start) as weekly_growth\n from weekly_jobs_list_cost\n order by week_start asc\n)\nselect\n week_start,\n weekly_growth,\n avg(weekly_growth) over (order by week_start ROWS BETWEEN 12 preceding AND CURRENT ROW) as 3_months_growth_moving_avg,\n list_cost\nfrom list_cost_growth"},{"name":"3b0096b1","displayName":"jobs_stale_datasets","query":"with producers as (\n SELECT \n workspace_id, entity_id as job_id, target_table_full_name, created_by\n FROM\n system.access.table_lineage\n WHERE\n entity_type=\"JOB\" AND \n target_table_full_name IS NOT NULL AND\n event_date > CURRENT_DATE() - INTERVAL 30 DAY\n GROUP BY ALL\n),\nconsumers as (\n SELECT \n workspace_id, source_table_full_name\n FROM\n system.access.table_lineage\n WHERE\n source_table_full_name IS NOT NULL AND event_date > CURRENT_DATE() - INTERVAL 30 DAY\n GROUP BY ALL\n),\njobs_with_datasets as (\n SELECT \n workspace_id, entity_id as job_id, collect_set(target_table_full_name) as populated_sets\n FROM\n system.access.table_lineage\n WHERE\n entity_type=\"JOB\" AND \n target_table_full_name IS NOT NULL\n GROUP BY ALL\n),\njobs_without_consumption as (\n SELECT \n t1.*\n FROM producers t1\n LEFT OUTER JOIN consumers t2\n ON t1.target_table_full_name = t2.source_table_full_name AND t1.workspace_id=t2.workspace_id\n),\njob_run_timeline_with_cost as (\n SELECT\n t1.*,\n t1.usage_metadata.job_id as job_id,\n t1.usage_metadata.job_run_id as run_id,\n t1.identity_metadata.run_as as run_as,\n t1.usage_quantity * list_prices.pricing.default AS list_cost\n FROM system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices \n ON\n t1.cloud = list_prices.cloud AND\n t1.sku_name = list_prices.sku_name AND\n t1.usage_start_time >= list_prices.price_start_time AND\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is NULL)\n WHERE\n t1.sku_name LIKE '%JOBS%' AND\n t1.usage_metadata.job_id IS NOT NULL AND\n t1.usage_metadata.job_run_id IS NOT NULL\n),\nagg_cost as (\n SELECT \n workspace_id,\n job_id,\n FIRST(run_as, TRUE) as run_as,\n SUM(list_cost) as list_cost\n FROM job_run_timeline_with_cost\n WHERE\n usage_start_time >= CURRENT_DATE() - INTERVAL 30 DAYS \n GROUP BY ALL\n ORDER BY list_cost DESC\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT\n t1.workspace_id,\n t5.name, \n t1.job_id,\n collect_set(target_table_full_name) as not_used_datasets,\n array_except(first(t3.populated_sets), collect_set(target_table_full_name)) as used_datasets,\n first(t4.run_as) as run_as,\n date(MAX(t2.period_end_time)) as last_seen_job_date,\n COUNT(DISTINCT t2.run_id) as runs,\n first(t4.list_cost) as list_cost_30d\nFROM \n jobs_without_consumption t1 \n INNER JOIN jobs_with_datasets t3 USING (workspace_id, job_id)\n LEFT JOIN system.lakeflow.job_run_timeline t2 USING (workspace_id, job_id)\n LEFT JOIN agg_cost t4 USING (workspace_id, job_id)\n LEFT JOIN most_recent_jobs t5 USING (workspace_id, job_id)\nGROUP BY ALL\nHAVING runs > 0 -- Exclude old Lineage entries for which we don't have jobs data\nORDER BY list_cost_30d DESC, runs DESC"},{"name":"d0be2fcd","displayName":"highest_change_jobs_7_14_days","query":"with job_run_timeline_with_cost as (\n SELECT\n t1.*,\n t1.usage_metadata.job_id as job_id,\n t1.identity_metadata.run_as as run_as,\n t1.usage_quantity * list_prices.pricing.default AS list_cost\n FROM system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices \n ON\n t1.cloud = list_prices.cloud AND\n t1.sku_name = list_prices.sku_name AND\n t1.usage_start_time >= list_prices.price_start_time AND\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is NULL)\n WHERE\n t1.sku_name LIKE '%JOBS%' AND\n t1.usage_metadata.job_id IS NOT NULL AND\n t1.usage_metadata.job_run_id IS NOT NULL AND\n t1.usage_date >= CURRENT_DATE() - INTERVAL 14 DAY\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT\n t2.name\n ,t1.workspace_id\n ,t1.job_id\n ,t1.sku_name\n ,t1.run_as\n ,Last7DaySpend\n ,Last14DaySpend\n ,last7DaySpend - last14DaySpend as Last7DayGrowth\n ,try_divide( (last7DaySpend - last14DaySpend) , last14DaySpend) * 100 AS Last7DayGrowthPct\nFROM\n (\n SELECT\n workspace_id,\n job_id,\n run_as,\n sku_name,\n SUM(list_cost) AS spend\n ,SUM(CASE WHEN usage_end_time BETWEEN date_add(current_date(), -8) AND date_add(current_date(), -1) THEN list_cost ELSE 0 END) AS Last7DaySpend\n ,SUM(CASE WHEN usage_end_time BETWEEN date_add(current_date(), -15) AND date_add(current_date(), -8) THEN list_cost ELSE 0 END) AS Last14DaySpend\n FROM job_run_timeline_with_cost\n GROUP BY ALL\n ) t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nORDER BY \n Last7DayGrowth DESC \nLIMIT 100"},{"name":"ec3ae3de","displayName":"cost_per_status_jobs_30_days","query":"with job_run_timeline_with_cost as (\n SELECT\n t1.cloud,\n t1.workspace_id,\n t2.job_id,\n t2.run_id,\n t2.result_state,\n t1.usage_start_time,\n t1.usage_end_time,\n t1.sku_name,\n t1.usage_unit,\n t1.usage_quantity,\n t1.usage_quantity * list_prices.pricing.default AS list_cost\n FROM system.billing.usage t1\n INNER JOIN system.lakeflow.job_run_timeline t2\n ON\n t1.workspace_id=t2.workspace_id\n AND t1.usage_metadata.job_id = t2.job_id\n AND t1.usage_metadata.job_run_id = t2.run_id\n AND t1.usage_start_time >= date_trunc(\"Hour\", t2.period_start_time)\n AND t1.usage_start_time < date_trunc(\"Hour\", t2.period_end_time) + INTERVAL 1 HOUR\n INNER JOIN system.billing.list_prices list_prices \n ON\n t1.cloud = list_prices.cloud AND\n t1.sku_name = list_prices.sku_name AND\n t1.usage_start_time >= list_prices.price_start_time AND\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is NULL)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n),\ncumulative_run_status_cost as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n result_state,\n usage_start_time,\n usage_end_time,\n SUM(list_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_cost\n FROM job_run_timeline_with_cost\n ORDER BY workspace_id, job_id, run_id, usage_end_time\n )\nSELECT\n workspace_id, \n job_id,\n run_id,\n result_state,\n usage_start_time,\n usage_end_time,\n cumulative_cost - COALESCE(LAG(cumulative_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time), 0) AS result_state_cost\nFROM cumulative_run_status_cost\nWHERE result_state IS NOT NULL\nORDER BY workspace_id, job_id, run_id, usage_end_time"},{"name":"26731098","displayName":"job_run_cost_agg_30_days","query":"with jobs_usage_with_list_cost as (\n SELECT\n t1.*,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%' \n AND usage_date >= CURRENT_DATE() - INTERVAL 30 DAYS \n),\njobs_usage_per_workspace as (\n SELECT\n workspace_id, \n SUM(list_cost) as list_cost\n FROM jobs_usage_with_list_cost\n GROUP BY ALL\n)\nSELECT \n t1.workspace_id,\n COUNT(DISTINCT t1.job_id) as num_jobs,\n COUNT(DISTINCT t1.run_id) as num_runs,\n first(t2.list_cost) as list_cost\nFROM system.lakeflow.job_run_timeline t1\n LEFT JOIN jobs_usage_per_workspace t2 USING (workspace_id)\nWHERE\n period_start_time >= CURRENT_DATE() - INTERVAL 30 DAYS \nGROUP BY ALL\nORDER BY list_cost DESC\n"},{"name":"07398605","displayName":"job_run_cost","query":"SELECT\n t1.*,\n t1.usage_metadata.job_id as job_id,\n t1.usage_metadata.job_run_id as run_id,\n t1.identity_metadata.run_as as run_as,\n t1.usage_quantity * list_prices.pricing.default AS list_cost\nFROM system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices \n ON\n t1.cloud = list_prices.cloud AND\n t1.sku_name = list_prices.sku_name AND\n t1.usage_start_time >= list_prices.price_start_time AND\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is NULL)\nWHERE\n t1.sku_name LIKE '%JOBS%' AND\n t1.usage_metadata.job_id IS NOT NULL AND \n t1.usage_metadata.job_run_id IS NOT NULL"},{"name":"d9fed998","displayName":"most_expensive_jobs_30_days","query":"with list_cost_per_job as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC"},{"name":"78ad8eef","displayName":"outlier_job_runs_30_days","query":"-- SPENDING BROKEN DOWN BY JOB\nwith stats_per_job_run as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.job_run_id as run_id,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen,\n MIN(t1.usage_start_time) as first_seen\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_metadata.job_run_id IS NOT NULL\n AND t1.usage_start_time >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY 1,2,3\n), \nstats_per_job AS (\n SELECT\n workspace_id,\n job_id,\n COUNT(run_id) AS runs,\n SUM(list_cost) AS total_list_cost,\n AVG(list_cost) AS avg_list_cost,\n MAX(list_cost) AS max_list_cost,\n percentile(list_cost, 0.9) AS p90\n FROM\n stats_per_job_run\n WHERE\n first_seen > CURRENT_TIMESTAMP() - INTERVAL 30 DAYS\n GROUP BY ALL\n ORDER BY\n total_list_cost DESC\n)\nSELECT\n *,\n (max_list_cost - p90) AS list_cost_deviation_from_p90\nFROM\n stats_per_job\nORDER BY\n list_cost_deviation_from_p90 DESC"},{"name":"0655a5b3","displayName":"30_60_days_spend","query":"with jobs_usage_with_list_cost as (\n SELECT\n t1.*,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n)\nSELECT\n *\n ,try_divide( (last30DaySpend - last30to60DaySpend) , last30to60DaySpend) * 100 AS Last30DayGrowth\nFROM\n (\n SELECT\n SUM(list_cost) AS spend\n ,sku_name\n ,SUM(CASE WHEN usage_end_time BETWEEN date_add(current_date(), -30) AND current_date() THEN list_cost ELSE 0 END) AS Last30DaySpend\n ,SUM(CASE WHEN usage_end_time BETWEEN date_add(current_date(), -60) AND date_add(current_date(), -31) THEN list_cost ELSE 0 END) AS Last30to60DaySpend\n FROM jobs_usage_with_list_cost\n GROUP BY ALL\n )"},{"name":"6fa8cef7","displayName":"most_expensive_runs_30_days","query":"with list_cost_per_job_run as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.job_run_id as run_id,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_metadata.job_run_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\n\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT \n t1.workspace_id,\n t2.name,\n t1.job_id,\n t1.run_id,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_run t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC"},{"name":"732b1d02","displayName":"highest_failure_jobs_30_days","query":"with job_run_timeline_with_cost as (\n SELECT\n t1.*,\n t1.identity_metadata.run_as as run_as,\n t2.job_id,\n t2.run_id,\n t2.result_state,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1\n INNER JOIN system.lakeflow.job_run_timeline t2\n ON\n t1.workspace_id=t2.workspace_id\n AND t1.usage_metadata.job_id = t2.job_id\n AND t1.usage_metadata.job_run_id = t2.run_id\n AND t1.usage_start_time >= date_trunc(\"Hour\", t2.period_start_time)\n AND t1.usage_start_time < date_trunc(\"Hour\", t2.period_end_time) + INTERVAL 1 HOUR\n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%' AND \n t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAYS\n),\ncumulative_run_status_cost as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n run_as,\n result_state,\n usage_end_time,\n SUM(list_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_cost\n FROM job_run_timeline_with_cost\n ORDER BY workspace_id, job_id, run_id, usage_end_time\n),\ncost_per_status as (\n SELECT\n workspace_id, \n job_id,\n run_id,\n run_as,\n result_state,\n usage_end_time,\n cumulative_cost - COALESCE(LAG(cumulative_cost) OVER (ORDER BY workspace_id, job_id, run_id, usage_end_time), 0) AS result_state_cost\n FROM cumulative_run_status_cost\n WHERE result_state IS NOT NULL\n ORDER BY workspace_id, job_id, run_id, usage_end_time),\ncost_per_status_agg as (\n SELECT\n workspace_id, \n job_id,\n FIRST(run_as, TRUE) as run_as,\n SUM(result_state_cost) as list_cost\n FROM cost_per_status\n WHERE\n result_state IN ('ERROR', 'FAILED', 'TIMED_OUT')\n GROUP BY ALL\n),\nterminal_statues as (\n SELECT\n workspace_id,\n job_id,\n CASE WHEN result_state IN ('ERROR', 'FAILED', 'TIMED_OUT') THEN 1 ELSE 0 END as is_failure,\n period_end_time as last_seen_date\n FROM system.lakeflow.job_run_timeline\n WHERE \n result_state IS NOT NULL AND\n period_end_time >= CURRENT_DATE() - INTERVAL 30 DAYS\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT\n first(t2.name) as name, \n t1.workspace_id,\n t1.job_id,\n COUNT(*) as runs,\n t3.run_as,\n SUM(is_failure) as failures,\n (1 - COALESCE(try_divide(SUM(is_failure), COUNT(*)), 0)) * 100 as success_ratio,\n first(t3.list_cost) as failure_list_cost,\n MAX(t1.last_seen_date) as last_seen_date\nFROM terminal_statues t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\n LEFT JOIN cost_per_status_agg t3 USING (workspace_id, job_id)\nGROUP BY ALL \nORDER BY failures DESC"},{"name":"2dbf594a","displayName":"job_cost_savings","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\tand t1.sku_name not ilike '%jobs_serverless%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\njobsrollingmonth as (\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC ),\n\nclusterutil as (SELECT * from \n\tsystem.compute.node_timeline\nWHERE\n\tstart_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\n\njobswithutil as (\nselect jobsrollingmonth.workspace_id, jobsrollingmonth.name,\navg(cpu_user_percent + cpu_system_percent) as `Avg CPU Utilization`, \nmax(cpu_user_percent + cpu_system_percent) as `Peak CPU Utilization`, \n\tavg(cpu_wait_percent) as `Avg CPU Disk Wait`,\n\tmax(cpu_wait_percent) as `Max CPU Disk Wait`,\n\tavg(mem_used_percent) as `Avg Memory Utilization`, \n\tmax(mem_used_percent) as `Max Memory Utilization`\nfrom clusterutil \njoin jobsrollingmonth\non clusterutil.cluster_id = jobsrollingmonth.cluster_id\ngroup by 1, 2),\n\nfinalutil as (\n select jobswithutil.*, sum(jobsrollingmonth.list_cost) as jobcostrolling30\n from jobswithutil\n join jobsrollingmonth\n on jobswithutil.workspace_id = jobsrollingmonth.workspace_id\n AND jobswithutil.name = jobsrollingmonth.name \n group by ALL\n)\n\nselect *, ((jobcostrolling30 * (1 - (`Avg CPU Utilization` / 100))) / 2.0) as maxsavings\nfrom finalutil\norder by maxsavings desc\nlimit 25"},{"name":"6d9e699b","displayName":"job_low_cpu","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\tand t1.sku_name not ilike '%jobs_serverless%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\njobsrollingmonth as (\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC ),\n\nclusterutil as (SELECT * from \n\tsystem.compute.node_timeline\nWHERE\n\tstart_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\n\njobswithutil as (\nselect jobsrollingmonth.workspace_id, jobsrollingmonth.name,\navg(cpu_user_percent + cpu_system_percent) as `Avg CPU Utilization`, \nmax(cpu_user_percent + cpu_system_percent) as `Peak CPU Utilization`, \n\tavg(cpu_wait_percent) as `Avg CPU Disk Wait`,\n\tmax(cpu_wait_percent) as `Max CPU Disk Wait`,\n\tavg(mem_used_percent) as `Avg Memory Utilization`, \n\tmax(mem_used_percent) as `Max Memory Utilization`\nfrom clusterutil \njoin jobsrollingmonth\non clusterutil.cluster_id = jobsrollingmonth.cluster_id\ngroup by 1, 2)\n\nselect *\nfrom jobswithutil\norder by `Avg CPU Utilization` asc\nlimit 10"},{"name":"f32e3463","displayName":"job_high_mem","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\tand t1.sku_name not ilike '%jobs_serverless%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\njobsrollingmonth as (\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC ),\n\nclusterutil as (SELECT * from \n\tsystem.compute.node_timeline\nWHERE\n\tstart_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\n\njobswithutil as (\nselect jobsrollingmonth.workspace_id, jobsrollingmonth.name,\navg(cpu_user_percent + cpu_system_percent) as `Avg CPU Utilization`, \nmax(cpu_user_percent + cpu_system_percent) as `Peak CPU Utilization`, \n\tavg(cpu_wait_percent) as `Avg CPU Disk Wait`,\n\tmax(cpu_wait_percent) as `Max CPU Disk Wait`,\n\tavg(mem_used_percent) as `Avg Memory Utilization`, \n\tmax(mem_used_percent) as `Max Memory Utilization`\nfrom clusterutil \njoin jobsrollingmonth\non clusterutil.cluster_id = jobsrollingmonth.cluster_id\ngroup by 1, 2)\n\nselect *\nfrom jobswithutil\norder by `Avg Memory Utilization` desc\nlimit 10"},{"name":"11611f07","displayName":"spend_by_tag","query":"with tagraw as (\n select explode(custom_tags) as (llave, val), (usage_quantity * list_prices.pricing.default)\n as list_cost \n from system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\t and t1.sku_name not ilike '%jobs_serverless%'\n and t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY),\n\ntagspend as ( select concat(llave,':',' ', val) as tagbucket, sum(list_cost) as list_cost\nfrom tagraw \ngroup by 1),\n\ncatchall as (select 'catchall' as tagbucket , sum(usage_quantity * list_prices.pricing.default)\nas list_cost \n from system.billing.usage t1\n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\t and t1.sku_name not ilike '%jobs_serverless%'\n and t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY\n and cardinality(custom_tags) = 0 )\n\nselect * from tagspend\nunion all\nselect * from catchall\norder by list_cost desc\nlimit 20"},{"name":"eda36d96","displayName":"job_low_dbr","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\tand t1.sku_name not ilike '%jobs_serverless%'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\njobsrollingmonth as (\nSELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\nFROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY ALL\nORDER BY list_cost DESC ),\n\nclusters as (SELECT * from \n\tsystem.compute.clusters\nWHERE\n\tdelete_time is not null\n)\n\nselect clusters.workspace_id, jobsrollingmonth.name, jobsrollingmonth.job_id,\njobsrollingmonth.run_as, sum(jobsrollingmonth.list_cost), any_value(dbr_Version) from \nclusters join jobsrollingmonth\non clusters.cluster_id = jobsrollingmonth.cluster_id\nwhere left(dbr_version, 1) <> '1'\ngroup by 1, 2, 3, 4\n"},{"name":"8aecbfa9","displayName":"job_no_scaling","query":"with list_cost_per_job_cluster as (\n SELECT\n t1.workspace_id,\n t1.usage_metadata.job_id,\n t1.usage_metadata.cluster_id,\n COUNT(DISTINCT t1.usage_metadata.job_run_id) as runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(identity_metadata.run_as, true) as run_as,\n first(t1.custom_tags, true) as custom_tags,\n MAX(t1.usage_end_time) as last_seen_date\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%'\n\t AND t1.sku_name NOT LIKE '%SERVERLESS'\n AND t1.usage_metadata.job_id IS NOT NULL\n AND t1.usage_metadata.cluster_id IS NOT NULL\n AND t1.usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n GROUP BY ALL\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\njob_runs_enhanced as (\n SELECT \n t2.name,\n t1.job_id,\n t1.workspace_id,\n t1.cluster_id,\n t1.runs,\n t1.run_as,\n SUM(list_cost) as list_cost,\n t1.last_seen_date\n FROM list_cost_per_job_cluster t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\n GROUP BY ALL\n ORDER BY list_cost DESC\n),\nclusters as (\n SELECT \n *,\n ROW_NUMBER() OVER(PARTITION BY workspace_id, cluster_id ORDER BY change_time DESC) as rn\n FROM \n system.compute.clusters\n WHERE\n cluster_source=\"JOB\"\n QUALIFY rn=1\n)\nSELECT\n clusters.workspace_id,\n jobs.job_id,\n first(jobs.name) as name,\n first(jobs.run_as) as run_as,\n SUM(jobs.list_cost) as list_cost,\n first(worker_count) as worker_count,\n first(min_autoscale_workers) as min_autoscale_workers,\n first(max_autoscale_workers) as max_autoscale_workers\nFROM clusters\n JOIN job_runs_enhanced jobs USING(workspace_id, cluster_id)\nWHERE\n (min_autoscale_workers is NULL OR max_autoscale_workers is NULL)\n and worker_count > 1\nGROUP BY 1,2\nORDER BY list_cost DESC"},{"name":"82bd998a","displayName":"jobs_no_tags","query":"with most_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\n\nrunlast30 as (\n\n select * from system.lakeflow.job_run_timeline\n where period_end_time > CURRENT_DATE - INTERVAL 30 DAY\n)\n\n\nselect most_recent_jobs.workspace_id, name, any_value(most_recent_jobs.job_id), \nany_value(tags), any_value(run_as) from most_recent_jobs\njoin runlast30\non most_recent_jobs.job_id = runlast30.job_id\nwhere cardinality(tags) = 0 \ngroup by all\n"},{"name":"07c866cc","displayName":"jobs_ap_clusters","query":"with clusters AS (\n SELECT\n *,\n ROW_NUMBER() OVER(PARTITION BY workspace_id, cluster_id ORDER BY change_time DESC) as rn\n FROM system.compute.clusters\n WHERE cluster_source=\"UI\"\n QUALIFY rn=1\n),\njob_tasks_exploded AS (\n SELECT\n workspace_id,\n job_id,\n EXPLODE(compute_ids) as cluster_id\n FROM system.lakeflow.job_task_run_timeline\n WHERE period_start_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\nall_purpose_cluster_jobs AS (\n SELECT\n t1.*,\n t2.cluster_name,\n t2.owned_by,\n t2.dbr_version\n FROM job_tasks_exploded t1 \n INNER JOIN clusters t2 USING (workspace_id, cluster_id)\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n)\nSELECT \n t1.workspace_id,\n t1.job_id,\n first(t2.name, TRUE) as name,\n array_size(collect_list(t1.cluster_id)) as task_runs,\n collect_set(\n named_struct(\n 'cluster_id', t1.cluster_id,\n 'cluster_name', t1.cluster_name\n )\n ) as clusters_list\nFROM all_purpose_cluster_jobs t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\nGROUP BY t1.workspace_id, t1.job_id \nORDER BY task_runs DESC\nLIMIT 100;"},{"name":"bdc3301f","displayName":"ap_clusters_by_jobs","query":"with clusters AS (\n SELECT\n *,\n ROW_NUMBER() OVER(PARTITION BY workspace_id, cluster_id ORDER BY change_time DESC) as rn\n FROM system.compute.clusters\n WHERE cluster_source=\"UI\"\n QUALIFY rn=1\n),\njob_tasks_exploded AS (\n SELECT\n workspace_id,\n job_id,\n EXPLODE(compute_ids) as cluster_id\n FROM system.lakeflow.job_task_run_timeline\n WHERE period_start_time >= CURRENT_DATE() - INTERVAL 30 DAY \n),\nall_purpose_cluster_jobs AS (\n SELECT\n t1.*,\n t2.cluster_name,\n t2.owned_by,\n t2.dbr_version\n FROM job_tasks_exploded t1 \n INNER JOIN clusters t2 USING (workspace_id, cluster_id)\n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n-- )\n),\nap_clusters_by_jobs as (\n SELECT \n t1.workspace_id,\n t1.cluster_id,\n first(t1.cluster_name, TRUE) as cluster_name,\n array_size(collect_list(t1.cluster_id)) as task_runs,\n collect_set(\n named_struct(\n 'job_id', t1.job_id,\n 'job_name', t2.name\n )\n ) as jobs_list\n FROM all_purpose_cluster_jobs t1 \n LEFT JOIN most_recent_jobs t2 USING (workspace_id, job_id)\n GROUP BY t1.workspace_id, t1.cluster_id \n ORDER BY task_runs DESC\n LIMIT 100\n)\nSELECT\n t1.workspace_id,\n t1.usage_metadata.cluster_id,\n first(t2.cluster_name, TRUE) as cluster_name,\n first(t2.task_runs) as task_runs,\n SUM(t1.usage_quantity * list_prices.pricing.default) as list_cost,\n first(t2.jobs_list) as jobs_list,\n MAX(t1.usage_end_time) as last_seen_date\nFROM system.billing.usage t1 \nINNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\nINNER JOIN ap_clusters_by_jobs t2 ON t1.workspace_id=t2.workspace_id AND t1.usage_metadata.cluster_id=t2.cluster_id\nWHERE usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \nGROUP BY ALL\nORDER BY list_cost DESC"},{"name":"ebc29bad","displayName":"usage_with_time","query":"WITH agg_data AS (\nSELECT ds, usage_start_time, usage_end_time,\n sku1 as sku,\n workspace_id,\n SUM(dbus) AS dbus\nFROM (select workspace_id, ds, dbus, sku, usage_start_time, usage_end_time, CASE\n WHEN sku LIKE '%ALL_PURPOSE%' THEN 'ALL_PURPOSE'\n WHEN sku LIKE '%JOBS%' THEN 'JOBS'\n WHEN sku LIKE '%DLT%' THEN 'DLT'\n WHEN sku LIKE '%SQL%' THEN 'SQL'\n WHEN sku LIKE '%INFERENCE%' THEN 'MODEL_INFERENCE'\n ELSE 'OTHER'\n END AS sku1 from (\nSELECT u.workspace_id, \n u.usage_date AS ds, \n u.sku_name AS sku, \n usage_start_time, usage_end_time,\n CAST(u.usage_quantity AS DOUBLE) AS dbus\nFROM system.billing.usage u \nWHERE u.usage_unit = 'DBU'\n))\nGROUP BY all)\nselect * from agg_data \norder by ds desc\n"},{"name":"994215cc","displayName":"expensive_job_analysis","query":"with jobs_usage_with_list_cost as (\n SELECT\n t1.*,\n t1.usage_quantity * list_prices.pricing.default as list_cost\n FROM system.billing.usage t1 \n INNER JOIN system.billing.list_prices list_prices on\n t1.cloud = list_prices.cloud and\n t1.sku_name = list_prices.sku_name and\n t1.usage_start_time >= list_prices.price_start_time and\n (t1.usage_end_time <= list_prices.price_end_time or list_prices.price_end_time is null)\n WHERE\n t1.sku_name LIKE '%JOBS%' \n),\nmost_expensive_jobs_30_days as (\n select \n usage_metadata.job_id as job_id,\n SUM(list_cost) as list_cost\n from\n jobs_usage_with_list_cost\n where usage_metadata.job_id is not null\n AND usage_date >= CURRENT_DATE() - INTERVAL 30 DAY \n group by 1\n order by list_cost desc limit :parameter \n),\nmost_recent_jobs as (\n SELECT \n *, \n ROW_NUMBER() OVER(PARTITION BY workspace_id, job_id ORDER BY change_time DESC) as rn\n FROM\n system.lakeflow.jobs QUALIFY rn=1\n),\nweekly_jobs_list_cost as (\n select\n t1.usage_start_time, t1.usage_end_time,\n t1.workspace_id,\n t1.usage_metadata.job_id as job_id,\n ifnull(t2.name, concat('JobId-', t1.usage_metadata.job_id)) as name,\n t1.sku_name,\n IF(size(t1.custom_tags) = 0 OR t1.custom_tags IS NULL, \n 'Untagged', \n concat_ws(', ', transform(map_keys(t1.custom_tags), key -> concat(key, ':', t1.custom_tags[key])))\n ) AS tags,\n SUM(t1.list_cost) as list_cost,\n COUNT(t1.usage_metadata.job_run_id) as no_of_runs\n from\n jobs_usage_with_list_cost t1 \n LEFT JOIN most_recent_jobs t2 \n on t1.workspace_id = t2.workspace_id and t1.usage_metadata.job_id = t2.job_id\n where usage_metadata.job_id in (select job_id from most_expensive_jobs_30_days)\n group by all\n)\nselect * from weekly_jobs_list_cost;\n","parameters":[{"displayName":"parameter","keyword":"parameter","dataType":"INTEGER","defaultSelection":{"values":{"dataType":"INTEGER","values":[{"value":"5"}]}}}]}],"pages":[{"name":"3c10256d","displayName":"New Page","layout":[{"widget":{"name":"d22ea6fe","queries":[{"name":"main_query","query":{"datasetName":"cc789c72","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"name","expression":"`name`"},{"name":"repair_list_cost","expression":"`repair_list_cost`"},{"name":"repair_time_seconds","expression":"`repair_time_seconds`"},{"name":"repairs","expression":"`repairs`"},{"name":"run_as","expression":"`run_as`"},{"name":"run_id","expression":"`run_id`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":4,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"repairs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":5,"title":"repairs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"repairs"},{"fieldName":"repair_time_seconds","numberFormat":"0,","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":6,"title":"repair_time","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"description":"Repair time in seconds","displayName":"repair_time_seconds"},{"fieldName":"repair_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":7,"title":"repair_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"repair_list_cost"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Most Retried Runs Last 30 Days","showDescription":true,"description":"- NULL repair time means there was no successful repair (yet)\n- We include all the jobs, but the run_as/repair_$list are calculated only for these running on serverless or job clusters."}}},"position":{"x":0,"y":147,"width":6,"height":10}},{"widget":{"name":"6064a1e4","queries":[{"name":"main_query","query":{"datasetName":"0655a5b3","fields":[{"name":"sum(Last30to60DaySpend)","expression":"SUM(`Last30to60DaySpend`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"counter","encodings":{"value":{"fieldName":"sum(Last30to60DaySpend)","displayName":"Sum of Last30to60DaySpend"}},"frame":{"showTitle":true,"title":"30-60 Day $list Spend","showDescription":false}}},"position":{"x":0,"y":11,"width":1,"height":2}},{"widget":{"name":"148bf5e5","queries":[{"name":"main_query","query":{"datasetName":"732b1d02","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"name","expression":"`name`"},{"name":"job_id","expression":"`job_id`"},{"name":"run_as","expression":"`run_as`"},{"name":"runs","expression":"`runs`"},{"name":"failures","expression":"`failures`"},{"name":"success_ratio","expression":"`success_ratio`"},{"name":"failure_list_cost","expression":"`failure_list_cost`"},{"name":"last_seen_date","expression":"`last_seen_date`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":4,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"runs"},{"fieldName":"failures","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":5,"title":"failures","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"failures"},{"fieldName":"success_ratio","numberFormat":"0.0%","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"float","displayAs":"number","visible":true,"order":6,"title":"success_ratio","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"cellFormat":{"default":{"foregroundColor":null},"rules":[{"if":{"column":"success_ratio","fn":"<","literal":"95"},"value":{"foregroundColor":"#E92828"}}]},"displayName":"success_ratio"},{"fieldName":"failure_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":7,"title":"failure_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"failure_list_cost"},{"fieldName":"last_seen_date","dateTimeFormat":"YYYY-MM-DD","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"datetime","displayAs":"datetime","visible":true,"order":8,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"last_seen_date"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":true,"frame":{"showTitle":true,"title":"Highest Failure Jobs Last 30 Days","showDescription":true,"description":"We include all the jobs, but the run_as/failure_$list are calculated only for these running on serverless or job clusters."}}},"position":{"x":0,"y":138,"width":6,"height":9}},{"widget":{"name":"8db952ae","queries":[{"name":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ac8a32e144897747346d0718262_period_end_time","query":{"datasetName":"e5c48fa4","fields":[{"name":"period_end_time","expression":"`period_end_time`"},{"name":"period_end_time_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ac8a34b1a9bb0dbd4181546224e_usage_end_time","query":{"datasetName":"07398605","fields":[{"name":"usage_end_time","expression":"`usage_end_time`"},{"name":"usage_end_time_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ac8a3431fe6aebcf5f4e02a6304_usage_end_time","query":{"datasetName":"ec3ae3de","fields":[{"name":"usage_end_time","expression":"`usage_end_time`"},{"name":"usage_end_time_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6eddc193172ca254bfb189ea5ded_usage_end_time","query":{"datasetName":"ebc29bad","fields":[{"name":"usage_end_time","expression":"`usage_end_time`"},{"name":"usage_end_time_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ee0982814b08eaa36f88e2f5964_usage_end_time","query":{"datasetName":"994215cc","fields":[{"name":"usage_end_time","expression":"`usage_end_time`"},{"name":"usage_end_time_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-date-range-picker","encodings":{"fields":[{"fieldName":"period_end_time","displayName":"period_end_time","queryName":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ac8a32e144897747346d0718262_period_end_time"},{"fieldName":"usage_end_time","displayName":"usage_end_time","queryName":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ac8a34b1a9bb0dbd4181546224e_usage_end_time"},{"fieldName":"usage_end_time","displayName":"usage_end_time","queryName":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ac8a3431fe6aebcf5f4e02a6304_usage_end_time"},{"fieldName":"usage_end_time","displayName":"usage_end_time","queryName":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6eddc193172ca254bfb189ea5ded_usage_end_time"},{"fieldName":"usage_end_time","displayName":"usage_end_time","queryName":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ee0982814b08eaa36f88e2f5964_usage_end_time"}]},"frame":{"showTitle":true,"title":"Date Range"}}},"position":{"x":0,"y":7,"width":3,"height":2}},{"widget":{"name":"a8666a3f","queries":[{"name":"main_query","query":{"datasetName":"d9fed998","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"name","expression":"`name`"},{"name":"job_id","expression":"`job_id`"},{"name":"run_as","expression":"`run_as`"},{"name":"runs","expression":"`runs`"},{"name":"list_cost","expression":"`list_cost`"},{"name":"last_seen_date","expression":"`last_seen_date`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":4,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"runs"},{"fieldName":"list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":5,"title":"list_cost","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost"},{"fieldName":"last_seen_date","dateTimeFormat":"YYYY-MM-DD","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"datetime","displayAs":"datetime","visible":true,"order":6,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"last_seen_date"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":true,"frame":{"showTitle":true,"title":"Most Expensive Jobs Last 30 Days"}}},"position":{"x":0,"y":49,"width":6,"height":9}},{"widget":{"name":"9b26bb04","queries":[{"name":"main_query","query":{"datasetName":"ec3ae3de","fields":[{"name":"daily(usage_end_time)","expression":"DATE_TRUNC(\"DAY\", `usage_end_time`)"},{"name":"result_state","expression":"`result_state`"},{"name":"sum(result_state_cost)","expression":"SUM(`result_state_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"daily(usage_end_time)","scale":{"type":"temporal"},"displayName":"date"},"y":{"fieldName":"sum(result_state_cost)","scale":{"type":"quantitative"},"axis":{"title":"% of $list by run status"},"displayName":"% of $list by run status"},"color":{"fieldName":"result_state","scale":{"type":"categorical","mappings":[{"value":"SUCCEEDED","color":"#00A972"},{"value":"FAILED","color":"#FCA4A1"},{"value":"ERROR","color":"#FF3621"},{"value":"SKIPPED","color":"#8BCAE7"},{"value":"TIMED_OUT","color":"#919191"}]},"displayName":"result_state"}},"frame":{"showTitle":true,"title":"Daily $list by Run Status Distribution","showDescription":false,"description":""},"mark":{"layout":"percent-stack"}}},"position":{"x":3,"y":43,"width":3,"height":6}},{"widget":{"name":"20d6788a","queries":[{"name":"main_query","query":{"datasetName":"07398605","fields":[{"name":"sku_name","expression":"`sku_name`"},{"name":"daily(usage_end_time)","expression":"DATE_TRUNC(\"DAY\", `usage_end_time`)"},{"name":"sum(list_cost)","expression":"SUM(`list_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"daily(usage_end_time)","scale":{"type":"temporal"},"displayName":"date"},"y":{"fieldName":"sum(list_cost)","scale":{"type":"quantitative"},"displayName":"$DBU","axis":{"title":"$list"}},"color":{"fieldName":"sku_name","scale":{"type":"categorical"},"displayName":"sku_name"}},"frame":{"showTitle":true,"title":"Daily $list by SKU","showDescription":true,"description":""}}},"position":{"x":3,"y":25,"width":3,"height":6}},{"widget":{"name":"cf70eb64","queries":[{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766215f0a7ef0292360d6895_job_id","query":{"datasetName":"732b1d02","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee1607664184e811fbf8ae29c6f9b_job_id","query":{"datasetName":"e5c48fa4","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766618a3988d3d1574121eb9_job_id","query":{"datasetName":"d9fed998","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee16076681a21be8d1a87b12f7a7e_job_id","query":{"datasetName":"ec3ae3de","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766a1a36821c722538cd0b97_job_id","query":{"datasetName":"6fa8cef7","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766c1a238c191c3fcd462999_job_id","query":{"datasetName":"cc789c72","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160767514eaaa492671b6de5b24_job_id","query":{"datasetName":"d0be2fcd","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee1607677147a8912a7296ca363d6_job_id","query":{"datasetName":"07398605","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee6b68a0d1199bfe2a4a773bce474_job_id","query":{"datasetName":"3b0096b1","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eef8fde24c1d6aaed907d5e57845fa_job_id","query":{"datasetName":"78ad8eef","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee1607664184e811fbf8ae29c6f9b_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee1607677147a8912a7296ca363d6_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766618a3988d3d1574121eb9_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160767514eaaa492671b6de5b24_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766a1a36821c722538cd0b97_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee16076681a21be8d1a87b12f7a7e_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766215f0a7ef0292360d6895_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee160766c1a238c191c3fcd462999_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eee6b68a0d1199bfe2a4a773bce474_job_id"},{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01eee160765f1a2aafa39a09af295526/datasets/01eef8fde24c1d6aaed907d5e57845fa_job_id"}]},"frame":{"showTitle":true,"showDescription":false,"title":"Job Id"}}},"position":{"x":4,"y":7,"width":1,"height":2}},{"widget":{"name":"9db6e298","queries":[{"name":"main_query","query":{"datasetName":"6fa8cef7","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"last_seen_date","expression":"`last_seen_date`"},{"name":"list_cost","expression":"`list_cost`"},{"name":"name","expression":"`name`"},{"name":"run_as","expression":"`run_as`"},{"name":"run_id","expression":"`run_id`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100001,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100002,"title":"job_id","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100003,"title":"run_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100004,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100005,"title":"$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"description":"","displayName":"list_cost"},{"fieldName":"last_seen_date","dateTimeFormat":"YYYY-MM-DD HH:mm:ss.SSS","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"datetime","displayAs":"datetime","visible":true,"order":100006,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"last_seen_date"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":true,"frame":{"showTitle":true,"title":"Most Expensive Runs Last 30 Days"}}},"position":{"x":0,"y":58,"width":6,"height":9}},{"widget":{"name":"a89a7010","queries":[{"name":"main_query","query":{"datasetName":"07398605","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"daily(usage_end_time)","expression":"DATE_TRUNC(\"DAY\", `usage_end_time`)"},{"name":"sum(list_cost)","expression":"SUM(`list_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"daily(usage_end_time)","scale":{"type":"temporal"},"displayName":"date"},"y":{"fieldName":"sum(list_cost)","scale":{"type":"quantitative"},"displayName":"$DBU","axis":{"title":"$list"}},"color":{"fieldName":"workspace_id","scale":{"type":"categorical"},"displayName":"workspace_id"}},"frame":{"showTitle":true,"title":"Daily $list by Workspace"}}},"position":{"x":3,"y":9,"width":3,"height":6}},{"widget":{"name":"9e95503d","queries":[{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1115189aa1b763efc94c0c34_workspace_id","query":{"datasetName":"e5c48fa4","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1128131fb9e01cac845917fd_workspace_id","query":{"datasetName":"26731098","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112c1eec8f52d7f62b5747c9_workspace_id","query":{"datasetName":"d9fed998","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1122109b80eafe6a8ef44c4b_workspace_id","query":{"datasetName":"d0be2fcd","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1135149380718c60d46a1635_workspace_id","query":{"datasetName":"6fa8cef7","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11251513a8c486a1267b2242_workspace_id","query":{"datasetName":"ec3ae3de","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1138158697aa8f1c17fd4792_workspace_id","query":{"datasetName":"732b1d02","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11181b3285cbe1695cbcaf4b_workspace_id","query":{"datasetName":"cc789c72","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112f17c7886968566cca9912_workspace_id","query":{"datasetName":"78ad8eef","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d111e1120a7fdf215a6d76f6c_workspace_id","query":{"datasetName":"3b0096b1","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_workspace_id","query":{"datasetName":"07398605","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"workspace_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1115189aa1b763efc94c0c34_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1128131fb9e01cac845917fd_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112c1eec8f52d7f62b5747c9_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1122109b80eafe6a8ef44c4b_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1135149380718c60d46a1635_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11251513a8c486a1267b2242_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1138158697aa8f1c17fd4792_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11181b3285cbe1695cbcaf4b_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112f17c7886968566cca9912_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d111e1120a7fdf215a6d76f6c_workspace_id"},{"fieldName":"workspace_id","displayName":"workspace_id","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_workspace_id"}]},"frame":{"showTitle":true,"title":"Workspace Id"}}},"position":{"x":3,"y":7,"width":1,"height":2}},{"widget":{"name":"ed508f60","textbox_spec":"## Tutorial\n\n---\n\n### How to get a job by ID?\n1. Copy the job_id\n2. Navigate to the correct workspace\n3. Paste the job ID in a top search toolbar\n4. Press enter\n\n### How to get a job run by ID?\n1. Follow the *How to get a job by ID?* section\n2. Copy run ID\n3. Open *any run* on the job runs page\n4. Replace the run ID in the url after `/runs/` part\n 1. *https://sample-workspace.databricks.com/jobs/123456789/runs/**111213141516***"},"position":{"x":0,"y":190,"width":6,"height":6}},{"widget":{"name":"07ff9fe2","queries":[{"name":"main_query","query":{"datasetName":"0655a5b3","fields":[{"name":"sum(Last30DaySpend)","expression":"SUM(`Last30DaySpend`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"counter","encodings":{"value":{"fieldName":"sum(Last30DaySpend)","displayName":"Sum of Last30DaySpend"}},"frame":{"showTitle":true,"title":"30 Day $list Spend","showDescription":false,"description":""}}},"position":{"x":0,"y":9,"width":1,"height":2}},{"widget":{"name":"6af8564a","queries":[{"name":"main_query","query":{"datasetName":"0655a5b3","fields":[{"name":"Last30DayGrowth","expression":"`Last30DayGrowth`"}],"disaggregated":true}}],"spec":{"version":2,"widgetType":"counter","encodings":{"value":{"fieldName":"Last30DayGrowth","displayName":"Last30DayGrowth"}},"frame":{"showTitle":true,"title":"30 Day Growth %","showDescription":false}}},"position":{"x":0,"y":13,"width":1,"height":2}},{"widget":{"name":"0a7f14c8","queries":[{"name":"main_query","query":{"datasetName":"bb7e5c78","fields":[{"name":"3_months_growth_moving_avg","expression":"`3_months_growth_moving_avg`"},{"name":"week_start","expression":"`week_start`"},{"name":"weekly_growth","expression":"`weekly_growth`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"combo","encodings":{"x":{"fieldName":"week_start","scale":{"type":"temporal"},"displayName":"week_start"},"y":{"primary":{"fields":[{"fieldName":"weekly_growth","displayName":"weekly %change"}]},"secondary":{"fields":[{"fieldName":"3_months_growth_moving_avg","displayName":"3_months_growth_moving_avg"}]},"scale":{"type":"quantitative"}}},"frame":{"showTitle":true,"title":"WoW $list %change"}}},"position":{"x":0,"y":25,"width":3,"height":6}},{"widget":{"name":"da4cfd60","queries":[{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1138158697aa8f1c17fd4792_run_as","query":{"datasetName":"732b1d02","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112c1eec8f52d7f62b5747c9_run_as","query":{"datasetName":"d9fed998","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11181b3285cbe1695cbcaf4b_run_as","query":{"datasetName":"cc789c72","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1135149380718c60d46a1635_run_as","query":{"datasetName":"6fa8cef7","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1122109b80eafe6a8ef44c4b_run_as","query":{"datasetName":"d0be2fcd","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d111e1120a7fdf215a6d76f6c_run_as","query":{"datasetName":"3b0096b1","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_run_as","query":{"datasetName":"07398605","fields":[{"name":"run_as","expression":"`run_as`"},{"name":"run_as_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1138158697aa8f1c17fd4792_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112c1eec8f52d7f62b5747c9_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d11181b3285cbe1695cbcaf4b_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1135149380718c60d46a1635_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d1122109b80eafe6a8ef44c4b_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d111e1120a7fdf215a6d76f6c_run_as"},{"fieldName":"run_as","displayName":"run_as","queryName":"dashboards/01ef070d110715f2b6d3061b8bda89ea/datasets/01ef070d112a1a5bbfc5a713abaf44c8_run_as"}]},"frame":{"showTitle":true,"title":"Run as "}}},"position":{"x":5,"y":7,"width":1,"height":2}},{"widget":{"name":"e8fc41c4","textbox_spec":"## Jobs System Tables Dashboard v0.2\n\n--- \n\nThis dashboard operates on a list cost (**$list**). The list cost is calculated as `number of DBUs * list price per DBU at that moment in time`. Account-specific rates and discounts are excluded.\n\nWe will be evolving it over time by adding more example queries and visualizations. The latest version will always be available in the GitHub link below.\n\n#### Documentation \n- Jobs System Tables (Public Preview)\n- [Latest Dashboard version GitHub](https://github.com/databricks/tmm/blob/main/System-Tables-Demo/Jobs-PrPr/Jobs%20System%20Tables%20Dashboard.lvdash.json)\n\n#### Requirements\n- For the full experience, this dashboard requires select rights on **system.usage**, **system.compute**, **system.lakeflow** and **system.access.table_lineage** System Tables\n - [How to Enable System Table Schema](https://docs.databricks.com/en/administration-guide/system-tables/index.html#enable-system-table-schemas)\n - [Grant access to system tables](https://docs.databricks.com/en/administration-guide/system-tables/index.html#grant-access-to-system-tables)\n\n#### Debugging\nFor detailed debugging instructions (how to find a job/run by ID) check out the guidance at the bottom of this dashboard."},"position":{"x":0,"y":0,"width":6,"height":7}},{"widget":{"name":"36768384","queries":[{"name":"main_query","query":{"datasetName":"78ad8eef","fields":[{"name":"avg_list_cost","expression":"`avg_list_cost`"},{"name":"job_id","expression":"`job_id`"},{"name":"list_cost_deviation_from_p90","expression":"`list_cost_deviation_from_p90`"},{"name":"max_list_cost","expression":"`max_list_cost`"},{"name":"p90","expression":"`p90`"},{"name":"runs","expression":"`runs`"},{"name":"total_list_cost","expression":"`total_list_cost`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100001,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100002,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"runs"},{"fieldName":"total_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100003,"title":"total_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"total_list_cost"},{"fieldName":"avg_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100004,"title":"avg_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"avg_list_cost"},{"fieldName":"max_list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100005,"title":"max_$list","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"description":"","displayName":"max_list_cost"},{"fieldName":"p90","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"float","displayAs":"number","visible":true,"order":100006,"title":"p90","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"p90"},{"fieldName":"list_cost_deviation_from_p90","numberFormat":"0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"float","displayAs":"number","visible":true,"order":100007,"title":"$list_deviation_from_p90","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost_deviation_from_p90"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"showDescription":true,"title":"Jobs Outliers Last 30 Days","description":"Analyse the job's $list deviation across multiple runs. "}}},"position":{"x":0,"y":157,"width":6,"height":8}},{"widget":{"name":"17e125b5","queries":[{"name":"main_query","query":{"datasetName":"e5c48fa4","fields":[{"name":"countdistinct(run_id)","expression":"COUNT(DISTINCT `run_id`)"},{"name":"daily(period_end_time)","expression":"DATE_TRUNC(\"DAY\", `period_end_time`)"},{"name":"result_state","expression":"`result_state`"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"daily(period_end_time)","scale":{"type":"temporal"},"displayName":"date"},"y":{"fieldName":"countdistinct(run_id)","scale":{"type":"quantitative"},"axis":{"title":"% of jobs by run status"},"displayName":"% of jobs by run status"},"color":{"fieldName":"result_state","scale":{"type":"categorical","mappings":[{"value":"ERROR","color":"#FF3621"},{"value":"SUCCEEDED","color":"#00A972"},{"value":"SKIPPED","color":"#8BCAE7"},{"value":"FAILED","color":"#FCA4A1"},{"value":"TIMED_OUT","color":"#919191"}]},"displayName":"result_state"}},"frame":{"showTitle":true,"title":"Daily Run Status Distribution","showDescription":true,"description":""},"mark":{"layout":"percent-stack"}}},"position":{"x":0,"y":43,"width":3,"height":6}},{"widget":{"name":"ce74daa3","queries":[{"name":"main_query","query":{"datasetName":"3b0096b1","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"last_seen_job_date","expression":"`last_seen_job_date`"},{"name":"list_cost_30d","expression":"`list_cost_30d`"},{"name":"name","expression":"`name`"},{"name":"not_used_datasets","expression":"`not_used_datasets`"},{"name":"run_as","expression":"`run_as`"},{"name":"runs","expression":"`runs`"},{"name":"used_datasets","expression":"`used_datasets`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":4,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"runs"},{"fieldName":"not_used_datasets","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"complex","displayAs":"json","visible":true,"order":5,"title":"not_used_datasets","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"not_used_datasets"},{"fieldName":"used_datasets","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"complex","displayAs":"json","visible":true,"order":6,"title":"used_datasets","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"used_datasets"},{"fieldName":"last_seen_job_date","dateTimeFormat":"YYYY-MM-DD","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"date","displayAs":"datetime","visible":true,"order":7,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"last_seen_job_date"},{"fieldName":"list_cost_30d","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":8,"title":"$list_30d","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost_30d"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Stale Datasets produced by Jobs","showDescription":true,"description":"Identify datasets without consumers"}}},"position":{"x":0,"y":165,"width":6,"height":9}},{"widget":{"name":"db74fb6d","queries":[{"name":"main_query","query":{"datasetName":"d0be2fcd","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"name","expression":"`name`"},{"name":"job_id","expression":"`job_id`"},{"name":"run_as","expression":"`run_as`"},{"name":"Last7DaySpend","expression":"`Last7DaySpend`"},{"name":"Last14DaySpend","expression":"`Last14DaySpend`"},{"name":"Last7DayGrowth","expression":"`Last7DayGrowth`"},{"name":"Last7DayGrowthPct","expression":"`Last7DayGrowthPct`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"Last7DaySpend","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":4,"title":"Last7DaySpend","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"Last7DaySpend"},{"fieldName":"Last14DaySpend","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":5,"title":"Last14DaySpend","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"Last14DaySpend"},{"fieldName":"Last7DayGrowth","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":6,"title":"Last7DayGrowth","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"Last7DayGrowth"},{"fieldName":"Last7DayGrowthPct","numberFormat":"0.00%","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":7,"title":"Last7DayGrowthPct","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"cellFormat":{"default":{"foregroundColor":null},"rules":[{"if":{"column":"Last7DayGrowthPct","fn":">","literal":"30"},"value":{"foregroundColor":"#E92828"}}]},"displayName":"Last7DayGrowthPct"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"sku_name","type":"string","displayAs":"string","order":8,"title":"sku_name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":true,"frame":{"showTitle":true,"title":"Highest change in weekly $list (past 2 weeks)"}}},"position":{"x":0,"y":67,"width":6,"height":8}},{"widget":{"name":"33b731f3","queries":[{"name":"main_query","query":{"datasetName":"26731098","fields":[{"name":"num_jobs","expression":"`num_jobs`"},{"name":"num_runs","expression":"`num_runs`"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"num_jobs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100001,"title":"jobs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"num_jobs"},{"fieldName":"num_runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100002,"title":"runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"num_runs"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"list_cost","type":"decimal","displayAs":"number","order":100003,"title":"list_cost","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Most Expensive Workspaces Last 30 Days ","showDescription":true,"description":"Based on Jobs Clusters & Serverless Jobs Cost"}}},"position":{"x":1,"y":9,"width":2,"height":6}},{"widget":{"name":"8cd97f64","queries":[{"name":"main_query","query":{"datasetName":"2dbf594a","fields":[{"name":"name","expression":"`name`"},{"name":"sum(maxsavings)","expression":"SUM(`maxsavings`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"name","axis":{"hideLabels":true,"title":"job name"},"scale":{"type":"categorical","sort":{"by":"y-reversed"}},"displayName":"job name"},"y":{"fieldName":"sum(maxsavings)","scale":{"type":"quantitative"},"displayName":"Max savings $list"},"color":{"fieldName":"name","scale":{"type":"categorical"},"legend":{"hideTitle":false,"position":"right"},"displayName":"name"}},"frame":{"showTitle":true,"title":"Top 25 Jobs by Potential Savings per Month","showDescription":true,"description":"These jobs provide highest ROI for optimization because they tend to be both pricey to run and also not close to fully leveraging harward potential. The calculation is optimistic-- it assumes that if you are at 20% utilization you can get similar performance with 20% of the hardware. Total savings would likely be lower."}}},"position":{"x":0,"y":79,"width":6,"height":9}},{"widget":{"name":"1b0433f2","queries":[{"name":"main_query","query":{"datasetName":"6d9e699b","fields":[{"name":"name","expression":"`name`"},{"name":"median(Avg CPU Utilization)","expression":"PERCENTILE(`Avg CPU Utilization`, 0.5)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"name","axis":{"hideLabels":true,"title":"job name"},"scale":{"type":"categorical","sort":{"by":"y-reversed"}},"displayName":"job name"},"y":{"fieldName":"median(Avg CPU Utilization)","scale":{"type":"quantitative"},"displayName":"Avg CPU Utilization %"},"color":{"fieldName":"name","scale":{"type":"categorical"},"displayName":"name"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Top 10 Jobs with Lowest Avg CPU Utilization","showDescription":true,"description":"These jobs are not close to fully leveraging the hardware potential - consider reducing cores or removing single-threaded code/ manual repartitions."}}},"position":{"x":0,"y":88,"width":3,"height":7}},{"widget":{"name":"3ae9678c","queries":[{"name":"main_query","query":{"datasetName":"11611f07","fields":[{"name":"sum(list_cost)","expression":"SUM(`list_cost`)"},{"name":"tagbucket","expression":"`tagbucket`"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"pie","encodings":{"angle":{"fieldName":"sum(list_cost)","scale":{"type":"quantitative"},"displayName":"Sum of list_cost"},"color":{"fieldName":"tagbucket","scale":{"type":"categorical"},"displayName":"tagbucket"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Rolling 30 $list by Tag","showDescription":true,"description":""}}},"position":{"x":0,"y":15,"width":6,"height":10}},{"widget":{"name":"3d6d0242","queries":[{"name":"main_query","query":{"datasetName":"f32e3463","fields":[{"name":"name","expression":"`name`"},{"name":"percentile(Avg Memory Utilization, percentage=0_5)","expression":"PERCENTILE(`Avg Memory Utilization`, 0.5)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"name","scale":{"type":"categorical","sort":{"by":"y-reversed"}},"axis":{"title":"job name","hideLabels":true},"displayName":"job name"},"y":{"fieldName":"percentile(Avg Memory Utilization, percentage=0_5)","scale":{"type":"quantitative"},"axis":{"hideLabels":false},"displayName":"Avg Memory Utilization %"},"color":{"fieldName":"name","scale":{"type":"categorical"},"displayName":"name"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Top 10 Jobs with Highest Avg Memory Utilization","showDescription":true,"description":"These jobs could experience bottlenecks - consider increasing memory allocation. "}}},"position":{"x":3,"y":88,"width":3,"height":7}},{"widget":{"name":"a04ffbd1","queries":[{"name":"main_query","query":{"datasetName":"8aecbfa9","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"name","expression":"`name`"},{"name":"job_id","expression":"`job_id`"},{"name":"run_as","expression":"`run_as`"},{"name":"list_cost","expression":"`list_cost`"},{"name":"worker_count","expression":"`worker_count`"},{"name":"min_autoscale_workers","expression":"`min_autoscale_workers`"},{"name":"max_autoscale_workers","expression":"`max_autoscale_workers`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"run_as","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"run_as"},{"fieldName":"list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":4,"title":"list_cost","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost"},{"fieldName":"worker_count","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":5,"title":"worker_count","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"worker_count"},{"fieldName":"min_autoscale_workers","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":6,"title":"min_autoscale_workers","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"min_autoscale_workers"},{"fieldName":"max_autoscale_workers","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":7,"title":"max_autoscale_workers","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"max_autoscale_workers"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Jobs with Fixed Number of Workers Last 30 Days","showDescription":true,"description":"These jobs do not leverage autoscaling, consider adding it to optimise the resources utilisation."}}},"position":{"x":0,"y":95,"width":6,"height":9}},{"widget":{"name":"0cd4c599","queries":[{"name":"main_query","query":{"datasetName":"eda36d96","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"name","expression":"`name`"},{"name":"job_id","expression":"`job_id`"},{"name":"any_value(dbr_Version)","expression":"`any_value(dbr_Version)`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"any_value(dbr_Version)","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":3,"title":"DBR version","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"any_value(dbr_Version)"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"run_as","type":"string","displayAs":"string","order":4,"title":"run_as","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"sum(list_cost)","type":"decimal","displayAs":"number","order":5,"title":"sum(list_cost)","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Jobs Running on Outdated DBR Version Last 30 Days","showDescription":true,"description":"These jobs are on old runtimes - consider upgrading to ensure optimal performance and ability to use new features"}}},"position":{"x":0,"y":104,"width":6,"height":8}},{"widget":{"name":"fecc5ab7","textbox_spec":"## Job Clusters Analysis \n----\nThe below charts and tables enrich job system table data with compute system table data to identify jobs with low CPU utilization, jobs with high memory utilization, jobs w/o scaling or auto-terminate, and jobs on low DBRs. Further it provides a ballpark estimate of jobs with highest savings potential based on current spend and current utilization. \n\nFor the following charts to work you need to leverage:\n- lakeflow system tables\n- compute system tables, include node_timeline Private Preview"},"position":{"x":0,"y":75,"width":6,"height":4}},{"widget":{"name":"bb2b0bd6","textbox_spec":"## Jobs Operational Observability "},"position":{"x":0,"y":136,"width":6,"height":2}},{"widget":{"name":"e9496dc5","queries":[{"name":"main_query","query":{"datasetName":"82bd998a","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"any_value(job_id)","expression":"`any_value(job_id)`"},{"name":"name","expression":"`name`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":0,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"any_value(job_id)","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":1,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"any_value(job_id)"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":2,"title":"name","allowSearch":true,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"}]},"invisibleColumns":[{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"any_value(tags)","type":"complex","displayAs":"json","order":3,"title":"any_value(tags)","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false},{"booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"any_value(run_as)","type":"string","displayAs":"string","order":4,"title":"any_value(run_as)","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"showDescription":true,"title":"Untagged Jobs Last 30 Days","description":"These jobs are not tagged and have been run in last 30 days; consider tagging them for cost attribution"}}},"position":{"x":0,"y":112,"width":6,"height":8}},{"widget":{"name":"84f6107f","queries":[{"name":"main_query","query":{"datasetName":"07c866cc","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"job_id","expression":"`job_id`"},{"name":"name","expression":"`name`"},{"name":"task_runs","expression":"`task_runs`"},{"name":"clusters_list","expression":"`clusters_list`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"job_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100001,"title":"job_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"job_id"},{"fieldName":"name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100002,"title":"name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"name"},{"fieldName":"task_runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100003,"title":"task_runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"task_runs"},{"fieldName":"clusters_list","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"complex","displayAs":"json","visible":true,"order":100004,"title":"clusters_list","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"clusters_list"}]},"invisibleColumns":[],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":false,"withRowNumber":false,"frame":{"showTitle":true,"title":"Most Frequent Jobs on All Purpose Clusters Last 30 Days","showDescription":true,"description":"High frequency jobs executed on all purpose clusters might prevent them from shutting down."}}},"position":{"x":0,"y":120,"width":6,"height":8}},{"widget":{"name":"7c10bacb","queries":[{"name":"main_query","query":{"datasetName":"bdc3301f","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"cluster_id","expression":"`cluster_id`"},{"name":"cluster_name","expression":"`cluster_name`"},{"name":"task_runs","expression":"`task_runs`"},{"name":"list_cost","expression":"`list_cost`"},{"name":"jobs_list","expression":"`jobs_list`"}],"disaggregated":true}}],"spec":{"version":1,"widgetType":"table","encodings":{"columns":[{"fieldName":"workspace_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100000,"title":"workspace_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"workspace_id"},{"fieldName":"cluster_id","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100001,"title":"cluster_id","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"cluster_id"},{"fieldName":"cluster_name","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"string","displayAs":"string","visible":true,"order":100002,"title":"cluster_name","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"cluster_name"},{"fieldName":"task_runs","numberFormat":"0","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"integer","displayAs":"number","visible":true,"order":100003,"title":"task_runs","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"task_runs"},{"fieldName":"list_cost","numberFormat":"$0.00","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"decimal","displayAs":"number","visible":true,"order":100004,"title":"list_cost","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"list_cost"},{"fieldName":"jobs_list","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"type":"complex","displayAs":"json","visible":true,"order":100005,"title":"jobs_list","allowSearch":false,"alignContent":"left","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false,"displayName":"jobs_list"}]},"invisibleColumns":[{"dateTimeFormat":"YYYY-MM-DD HH:mm:ss.SSS","booleanValues":["false","true"],"imageUrlTemplate":"{{ @ }}","imageTitleTemplate":"{{ @ }}","imageWidth":"","imageHeight":"","linkUrlTemplate":"{{ @ }}","linkTextTemplate":"{{ @ }}","linkTitleTemplate":"{{ @ }}","linkOpenInNewTab":true,"name":"last_seen_date","type":"datetime","displayAs":"datetime","order":100006,"title":"last_seen_date","allowSearch":false,"alignContent":"right","allowHTML":false,"highlightLinks":false,"useMonospaceFont":false,"preserveWhitespace":false}],"allowHTMLByDefault":false,"itemsPerPage":25,"paginationSize":"default","condensed":true,"withRowNumber":false,"frame":{"showTitle":true,"title":"All Purpose Clusters with the Highest Job Run Frequency Last 30 Days","showDescription":true,"description":"List of your all purpose clusters most frequently used to run jobs on. "}}},"position":{"x":0,"y":128,"width":6,"height":8}},{"widget":{"name":"4551b787","queries":[{"name":"main_query","query":{"datasetName":"ebc29bad","fields":[{"name":"sku","expression":"`sku`"},{"name":"weekly(ds)","expression":"DATE_TRUNC(\"WEEK\", `ds`)"},{"name":"sum(dbus)","expression":"SUM(`dbus`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"weekly(ds)","scale":{"type":"temporal"},"displayName":"week"},"y":{"fieldName":"sum(dbus)","scale":{"type":"quantitative"},"displayName":"DBUs"},"color":{"fieldName":"sku","scale":{"type":"categorical"},"displayName":"sku"}},"frame":{"showTitle":true,"title":"Total DBU consumption with time"}}},"position":{"x":0,"y":31,"width":6,"height":6}},{"widget":{"name":"d0225af6","queries":[{"name":"main_query","query":{"datasetName":"ebc29bad","fields":[{"name":"sum(dbus)","expression":"SUM(`dbus`)"},{"name":"workspace_id","expression":"`workspace_id`"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"pie","encodings":{"angle":{"fieldName":"sum(dbus)","scale":{"type":"quantitative"},"displayName":"DBUs"},"color":{"fieldName":"workspace_id","scale":{"type":"categorical"},"displayName":"workspace_id"}},"frame":{"showTitle":true,"title":"DBUs across workspaces"}}},"position":{"x":0,"y":37,"width":3,"height":6}},{"widget":{"name":"207290d4","queries":[{"name":"main_query","query":{"datasetName":"ebc29bad","fields":[{"name":"workspace_id","expression":"`workspace_id`"},{"name":"weekly(ds)","expression":"DATE_TRUNC(\"WEEK\", `ds`)"},{"name":"sum(dbus)","expression":"SUM(`dbus`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"line","encodings":{"x":{"fieldName":"weekly(ds)","scale":{"type":"temporal"},"displayName":"week"},"y":{"fieldName":"sum(dbus)","scale":{"type":"quantitative"},"displayName":"Sum of dbus"},"color":{"fieldName":"workspace_id","scale":{"type":"categorical"},"displayName":"workspace_id"}},"frame":{"showTitle":true,"title":"DBU consumption per workspace"}}},"position":{"x":3,"y":37,"width":3,"height":6}},{"widget":{"name":"8b43d58a","textbox_spec":"## Analysis of Most Expensive Jobs"},"position":{"x":0,"y":174,"width":6,"height":2}},{"widget":{"name":"c86cc8e9","queries":[{"name":"main_query","query":{"datasetName":"994215cc","fields":[{"name":"name","expression":"`name`"},{"name":"weekly(usage_end_time)","expression":"DATE_TRUNC(\"WEEK\", `usage_end_time`)"},{"name":"sum(list_cost)","expression":"SUM(`list_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"line","encodings":{"x":{"fieldName":"weekly(usage_end_time)","scale":{"type":"temporal"},"displayName":"week"},"y":{"fieldName":"sum(list_cost)","scale":{"type":"quantitative"},"displayName":"$list"},"color":{"fieldName":"name","scale":{"type":"categorical"},"displayName":"Job name"}},"frame":{"showTitle":true,"title":"Job cost over time"}}},"position":{"x":0,"y":178,"width":3,"height":6}},{"widget":{"name":"13180704","queries":[{"name":"main_query","query":{"datasetName":"994215cc","fields":[{"name":"name","expression":"`name`"},{"name":"weekly(usage_end_time)","expression":"DATE_TRUNC(\"WEEK\", `usage_end_time`)"},{"name":"sum(no_of_runs)","expression":"SUM(`no_of_runs`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"line","encodings":{"x":{"fieldName":"weekly(usage_end_time)","scale":{"type":"temporal"},"displayName":"week"},"y":{"fieldName":"sum(no_of_runs)","scale":{"type":"quantitative"},"displayName":"runs"},"color":{"fieldName":"name","scale":{"type":"categorical"},"displayName":"Job name"}},"frame":{"showTitle":true,"title":"Job runs"}}},"position":{"x":3,"y":178,"width":3,"height":6}},{"widget":{"name":"2e780615","queries":[{"name":"main_query","query":{"datasetName":"994215cc","fields":[{"name":"sku_name","expression":"`sku_name`"},{"name":"weekly(usage_end_time)","expression":"DATE_TRUNC(\"WEEK\", `usage_end_time`)"},{"name":"sum(list_cost)","expression":"SUM(`list_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"bar","encodings":{"x":{"fieldName":"weekly(usage_end_time)","scale":{"type":"temporal"},"displayName":"week"},"y":{"fieldName":"sum(list_cost)","scale":{"type":"quantitative"},"displayName":"$list"},"color":{"fieldName":"sku_name","scale":{"type":"categorical"},"displayName":"SKU"}},"frame":{"showTitle":true,"title":"List cost per SKU"}}},"position":{"x":0,"y":184,"width":3,"height":6}},{"widget":{"name":"b5433db1","queries":[{"name":"main_query","query":{"datasetName":"994215cc","fields":[{"name":"tags","expression":"`tags`"},{"name":"weekly(usage_end_time)","expression":"DATE_TRUNC(\"WEEK\", `usage_end_time`)"},{"name":"sum(list_cost)","expression":"SUM(`list_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"line","encodings":{"x":{"fieldName":"weekly(usage_end_time)","scale":{"type":"temporal"},"displayName":"week"},"y":{"fieldName":"sum(list_cost)","scale":{"type":"quantitative"},"displayName":"$list"},"color":{"fieldName":"tags","scale":{"type":"categorical","mappings":[{"value":"{ds_team_name -> music}","color":"#FFAB00"}]},"displayName":"Tags"}},"frame":{"showTitle":true,"title":"List cost per tag"}}},"position":{"x":3,"y":184,"width":3,"height":6}},{"widget":{"name":"81d8fbeb","queries":[{"name":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ee0982814b08eaa36f88e2f5964_name","query":{"datasetName":"994215cc","fields":[{"name":"name","expression":"`name`"},{"name":"name_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"name","displayName":"name","queryName":"dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ee0982814b08eaa36f88e2f5964_name"}]},"frame":{"showTitle":true,"title":"Job Name"}}},"position":{"x":3,"y":176,"width":3,"height":2}},{"widget":{"name":"09ae24f5","queries":[{"name":"parameter_dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ee0982814b08eaa36f88e2f5964_parameter","query":{"datasetName":"994215cc","parameters":[{"name":"parameter","keyword":"parameter"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"parameterName":"parameter","queryName":"parameter_dashboards/01ef6ac8a3291e7585424710b6bf3ac7/datasets/01ef6ee0982814b08eaa36f88e2f5964_parameter"}]},"frame":{"showTitle":true,"title":"Number of jobs ","showDescription":true,"description":"Eg. showing 5 most expensive jobs in the last 30 days"}}},"position":{"x":0,"y":176,"width":3,"height":2}}]}]} \ No newline at end of file