diff --git a/docs/admin/code-hosts/aws-codecommit.mdx b/docs/admin/code-hosts/aws-codecommit.mdx index 5f1666ebc..a7fc693af 100644 --- a/docs/admin/code-hosts/aws-codecommit.mdx +++ b/docs/admin/code-hosts/aws-codecommit.mdx @@ -37,7 +37,7 @@ AWS CodeCommit connections support the following configuration options, which ar {/* SCHEMA_SYNC_START: admin/code_hosts/aws_codecommit.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { // REQUIRED: diff --git a/docs/admin/code-hosts/azuredevops.mdx b/docs/admin/code-hosts/azuredevops.mdx index 7695f459c..31c6dc495 100644 --- a/docs/admin/code-hosts/azuredevops.mdx +++ b/docs/admin/code-hosts/azuredevops.mdx @@ -69,7 +69,7 @@ Azure DevOps connections support the following configuration options, which are {/* SCHEMA_SYNC_START: admin/code_hosts/azuredevops.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json // Authentication alternatives: token OR windowsPassword diff --git a/docs/admin/code-hosts/bitbucket-cloud.mdx b/docs/admin/code-hosts/bitbucket-cloud.mdx index 51070727f..9ec51d65b 100644 --- a/docs/admin/code-hosts/bitbucket-cloud.mdx +++ b/docs/admin/code-hosts/bitbucket-cloud.mdx @@ -116,7 +116,7 @@ Bitbucket Cloud connections support the following configuration options, which a {/* SCHEMA_SYNC_START: admin/code_hosts/bitbucket_cloud.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { // The workspace access token to use when authenticating with Bitbucket Cloud. diff --git a/docs/admin/code-hosts/bitbucket-server.mdx b/docs/admin/code-hosts/bitbucket-server.mdx index 6775b7dbb..5ddea898e 100644 --- a/docs/admin/code-hosts/bitbucket-server.mdx +++ b/docs/admin/code-hosts/bitbucket-server.mdx @@ -202,7 +202,7 @@ Bitbucket Server / Bitbucket Data Center connections support the following confi {/* SCHEMA_SYNC_START: admin/code_hosts/bitbucket_server.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json // Authentication alternatives: token OR password diff --git a/docs/admin/code-hosts/gerrit.mdx b/docs/admin/code-hosts/gerrit.mdx index f20e399f3..ce9366139 100644 --- a/docs/admin/code-hosts/gerrit.mdx +++ b/docs/admin/code-hosts/gerrit.mdx @@ -113,7 +113,7 @@ Gerrit connections support the following configuration options, which are specif {/* SCHEMA_SYNC_START: admin/code_hosts/gerrit.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { // If non-null, enforces Gerrit repository permissions. This requires that there is an item in the [site configuration json](https://sourcegraph.com/docs/admin/config/site_config#auth-providers) `auth.providers` field, of type "gerrit" with the same `url` field as specified in this `GerritConnection`. diff --git a/docs/admin/code-hosts/github.mdx b/docs/admin/code-hosts/github.mdx index c13555016..ebb0d4257 100644 --- a/docs/admin/code-hosts/github.mdx +++ b/docs/admin/code-hosts/github.mdx @@ -458,7 +458,7 @@ GitHub connections support the following configuration options, which are specif {/* SCHEMA_SYNC_START: admin/code_hosts/github.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json // Authentication alternatives: token OR gitHubAppDetails OR externalAccount OR useRandomExternalAccount diff --git a/docs/admin/code-hosts/gitlab.mdx b/docs/admin/code-hosts/gitlab.mdx index 518ecea02..04f610404 100644 --- a/docs/admin/code-hosts/gitlab.mdx +++ b/docs/admin/code-hosts/gitlab.mdx @@ -189,7 +189,7 @@ See [Internal rate limits](/admin/code-hosts/rate-limits#internal-rate-limits). {/* SCHEMA_SYNC_START: admin/code_hosts/gitlab.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { // If non-null, enforces GitLab repository permissions. This requires that there be an item in the `auth.providers` field of type "gitlab" with the same `url` field as specified in this `GitLabConnection`. diff --git a/docs/admin/code-hosts/gitolite.mdx b/docs/admin/code-hosts/gitolite.mdx index f6837c8f2..85c2bd07e 100644 --- a/docs/admin/code-hosts/gitolite.mdx +++ b/docs/admin/code-hosts/gitolite.mdx @@ -25,7 +25,7 @@ To connect Gitolite to Sourcegraph: {/* SCHEMA_SYNC_START: admin/code_hosts/gitolite.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { // A list of repositories to never mirror from this Gitolite instance. Supports excluding by exact name ({"name": "foo"}). diff --git a/docs/admin/code-hosts/other.mdx b/docs/admin/code-hosts/other.mdx index f92d971c6..213c6cf75 100644 --- a/docs/admin/code-hosts/other.mdx +++ b/docs/admin/code-hosts/other.mdx @@ -68,7 +68,7 @@ Repositories must be listed individually: {/* SCHEMA_SYNC_START: admin/code_hosts/other_external_service.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { // A list of repositories to never mirror by name after applying repositoryPathPattern. Supports excluding by exact name ({"name": "myrepo"}) or regular expression ({"pattern": ".*secret.*"}). diff --git a/docs/admin/code-hosts/phabricator.mdx b/docs/admin/code-hosts/phabricator.mdx index 3f1c489b8..e43484067 100644 --- a/docs/admin/code-hosts/phabricator.mdx +++ b/docs/admin/code-hosts/phabricator.mdx @@ -76,7 +76,7 @@ The Sourcegraph instance's site admin must [update the `corsOrigin` site config {/* SCHEMA_SYNC_START: admin/code_hosts/phabricator.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { // SSH cipher to use when cloning via SSH. Must be a valid choice from `ssh -Q cipher`. diff --git a/docs/admin/config/settings.mdx b/docs/admin/config/settings.mdx index 7fd8d1822..2610a5649 100644 --- a/docs/admin/config/settings.mdx +++ b/docs/admin/config/settings.mdx @@ -27,7 +27,7 @@ Settings options and their default values are shown below. {/* SCHEMA_SYNC_START: admin/config/settings.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { diff --git a/docs/admin/config/site-config.mdx b/docs/admin/config/site-config.mdx index 6904c12de..7d08af7da 100644 --- a/docs/admin/config/site-config.mdx +++ b/docs/admin/config/site-config.mdx @@ -21,7 +21,7 @@ All site configuration options and their default values are shown below. {/* SCHEMA_SYNC_START: admin/config/site.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { @@ -305,7 +305,7 @@ All site configuration options and their default values are shown below. "exclude": null }, "deepSearch.enabled": false, - "deepSearch.model": "anthropic::2024-10-22::claude-sonnet-4-6-latest", + "deepSearch.model": null, "deepSearch.sharing.enabled": false, "deepSearch.showCodyWeb": false, "enableGithubInternalRepoVisibility": false, diff --git a/docs/admin/repo/perforce.mdx b/docs/admin/repo/perforce.mdx index 5adb427f1..68531400c 100644 --- a/docs/admin/repo/perforce.mdx +++ b/docs/admin/repo/perforce.mdx @@ -228,7 +228,7 @@ With this setting, Sourcegraph will ignore any rules with a host other than `*`, {/* SCHEMA_SYNC_START: admin/code_hosts/perforce.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-08-05T21:41:22Z */} +{/* Last updated: 2026-08-19T07:17:05Z */} ```json { // If non-null, enforces Perforce depot permissions. diff --git a/docs/cody/capabilities/supported-models.mdx b/docs/cody/capabilities/supported-models.mdx index b2d431cfe..9093c49f1 100644 --- a/docs/cody/capabilities/supported-models.mdx +++ b/docs/cody/capabilities/supported-models.mdx @@ -15,39 +15,16 @@ Cody supports a variety of cutting-edge large language models for use in chat an | **Provider** | **Model** | **Status** | **Vision Support** | | :----------- | :-------- | :--------- | :----------------- | | Anthropic | [Claude Opus 5](https://platform.claude.com/docs/en/about-claude/models/whats-new-opus-5) | ✅ | ✅ | -| Anthropic | [Claude Opus 4.8](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | -| Anthropic | [Claude Opus 4.7](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | -| Anthropic | [Claude Opus 4.6](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | | Anthropic | [Claude Sonnet 5](https://platform.claude.com/docs/en/about-claude/models/whats-new-sonnet-5) | ✅ | ✅ | -| Anthropic | [Claude Sonnet 4.6](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | -| Anthropic | [Claude Sonnet 4.6 with Thinking](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | -| Anthropic | [Claude Sonnet 4.5](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | -| Anthropic | [Claude Sonnet 4.5 with Thinking](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | -| Anthropic | [Claude Opus 4.5](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | -| Anthropic | [Claude Opus 4.5 with Thinking](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | | Anthropic | [Claude Haiku 4.5](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | | Anthropic | [Claude Haiku 4.5 with Thinking](https://platform.claude.com/docs/en/about-claude/models/overview) | ✅ | ✅ | -| Google | [Gemini 2.5 Flash](https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash) | ✅ | ✅ | -| Google | [Gemini 2.5 Pro](https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-pro) | ✅ | ✅ | -| Google | [Gemini 3.1 Flash Lite](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-1-flash-lite) | ✅ | ✅ | +| Google | [Gemini 3.7 Flash](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-7-flash) | ✅ | ✅ | +| Google | [Gemini 3.6 Flash](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-6-flash) | ✅ | ✅ | +| Google | [Gemini 3.5 Flash-Lite](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-5-flash-lite) | ✅ | ✅ | | Google | [Gemini 3.1 Pro](https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-1-pro) | ✅ (beta) | ❌ | -| Google | [Gemini 3.5 Flash](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-5-flash) | ✅ | ✅ | | OpenAI | [GPT-5.6 Sol](https://developers.openai.com/api/docs/models/gpt-5.6-sol) | ✅ | ✅ | | OpenAI | [GPT-5.6 Terra](https://developers.openai.com/api/docs/models/gpt-5.6-terra) | ✅ | ✅ | | OpenAI | [GPT-5.6 Luna](https://developers.openai.com/api/docs/models/gpt-5.6-luna) | ✅ | ✅ | -| OpenAI | [GPT-5.4](https://developers.openai.com/api/docs/models/gpt-5.4) | ✅ | ✅ | -| OpenAI | [GPT-5.4 mini](https://developers.openai.com/api/docs/models/gpt-5.4-mini) | ✅ | ✅ | -| OpenAI | [GPT-5.4 nano](https://developers.openai.com/api/docs/models/gpt-5.4-nano) | ✅ | ✅ | -| OpenAI | [GPT-5.2](https://platform.openai.com/docs/models/gpt-5.2) | ✅ | ✅ | -| OpenAI | [GPT-5.1](https://platform.openai.com/docs/models/gpt-5.1) | ✅ | ✅ | -| OpenAI | [GPT-5](https://platform.openai.com/docs/models/gpt-5) | ✅ | ✅ | -| OpenAI | [GPT-5 mini](https://platform.openai.com/docs/models/gpt-5-mini) | ✅ | ✅ | -| OpenAI | [GPT-5 nano](https://platform.openai.com/docs/models/gpt-5-nano) | ✅ | ✅ | -| OpenAI | [GPT-4o](https://platform.openai.com/docs/models#gpt-4o) | ✅ | ✅ | -| OpenAI | [GPT-4.1](https://platform.openai.com/docs/models/gpt-4.1) | ✅ | ✅ | -| OpenAI | [GPT-4o-mini](https://platform.openai.com/docs/models#gpt-4o-mini) | ✅ | ✅ | -| OpenAI | [GPT-4.1-mini](https://platform.openai.com/docs/models/gpt-4.1-mini) | ✅ | ✅ | -| OpenAI | [o3](https://platform.openai.com/docs/models#o3) | ✅ | ❌ | While Gemini models support vision capabilities, Cody clients do not @@ -78,6 +55,6 @@ The following models are used by default for each feature when no specific model | **Feature** | **Default Model** | | :---------- | :---------------- | -| Chat | Claude Sonnet 4.5 | +| Chat | Claude Sonnet 5 | | Autocomplete | DeepSeek V2 Lite Base | | Fast Chat | Claude Haiku 4.5 | diff --git a/docs/self-hosted/observability/alerts.mdx b/docs/self-hosted/observability/alerts.mdx index db6679c77..9fa037f60 100644 --- a/docs/self-hosted/observability/alerts.mdx +++ b/docs/self-hosted/observability/alerts.mdx @@ -3607,6 +3607,43 @@ Generated query for critical alert: `max((max(src_repoupdater_cleanup_failed_rep
+## worker: entitlements_usage_history_telemetry_queue_oldest_age + +

age of the oldest unexported entitlement usage history event

+ +**Descriptions** + +- warning worker: 1800s+ age of the oldest unexported entitlement usage history event for 30m0s +- critical worker: 7200s+ age of the oldest unexported entitlement usage history event for 2h0m0s + +**Next steps** + +- Check whether the entitlement usage history telemetry export job is failing or making no forward progress. +- Compare this panel with the exported events and export batch size panels above. +- Check worker logs for entitlements usage history telemetry exporter failures. +- More help interpreting this metric is available in the [dashboards reference](dashboards#worker-entitlements_usage_history_telemetry_queue_oldest_age). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_worker_entitlements_usage_history_telemetry_queue_oldest_age", + "critical_worker_entitlements_usage_history_telemetry_queue_oldest_age" +] +``` + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Generated query for warning alert: `max(((max(src_entitlementstelemetry_usage_history_queue_oldest_age_seconds and on (instance, job) ((increase(src_entitlementstelemetry_usage_history_queue_metrics_total{job=~"^worker.*"\}[15m]) > 0) or on (instance, job) (increase(src_entitlementstelemetry_usage_history_queue_metrics_errors_total\{job=~"^worker.*"}[15m]) > 0)))) or on () vector(0)) > 1800)` + +Generated query for critical alert: `max(((max(src_entitlementstelemetry_usage_history_queue_oldest_age_seconds and on (instance, job) ((increase(src_entitlementstelemetry_usage_history_queue_metrics_total{job=~"^worker.*"\}[15m]) > 0) or on (instance, job) (increase(src_entitlementstelemetry_usage_history_queue_metrics_errors_total\{job=~"^worker.*"}[15m]) > 0)))) or on () vector(0)) > 7200)` + +
+ +
+ ## worker: src_repoupdater_external_services_total

the total number of external services

@@ -6524,6 +6561,42 @@ Generated query for warning alert: `max((max by (sg_instance) (go_gc_duration_se
+## codeintel-uploads: codeintel_uploads_janitor_scip_documents_oldest_dereference_log_age + +

approximate age of the oldest SCIP document dereference log record, based on the lowest-ID record

+ +**Descriptions** + +- warning codeintel-uploads: 604800s+ approximate age of the oldest SCIP document dereference log record, based on the lowest-ID record for 15m0s + +**Next steps** + +- Check the SCIP documents janitor error rate and duration panels for failures or slow cleanup queries. +- Check sum(rate(src_codeintel_uploads_janitor_scip_documents_records_scanned_total[5m])). A sustained non-zero rate means the janitor is processing eligible dereference logs. +- Check sum(rate(src_codeintel_uploads_janitor_scip_documents_records_altered_total[5m])) for the document deletion rate. This may legitimately be lower than the scanned rate because referenced documents are retained. +- Check codeintel-db CPU, I/O, locks, and table growth for codeintel_scip_documents_dereference_logs. +- If cleanup is healthy but cannot keep up with the volume of generated logs, increase CODEINTEL_UPLOADS_UNREFERENCED_DOCUMENT_MAX_BATCHES. +- If increasing the batch budget is insufficient and codeintel-db has capacity, increase CODEINTEL_UPLOADS_UNREFERENCED_DOCUMENT_MAX_CONCURRENT_BATCHES. +- Learn more about the related dashboard panel in the [dashboards reference](dashboards#codeintel-uploads-codeintel_uploads_janitor_scip_documents_oldest_dereference_log_age). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_codeintel-uploads_codeintel_uploads_janitor_scip_documents_oldest_dereference_log_age" +] +``` + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Generated query for warning alert: `max(((time() - max(src_codeintel_uploads_janitor_scip_documents_oldest_dereference_log_timestamp_seconds)) and max(src_codeintel_uploads_janitor_scip_documents_oldest_dereference_log_timestamp_seconds) > 0) > 604800)` + +
+ +
+ ## telemetry: telemetry_gateway_exporter_queue_growth

rate of growth of events export queue over 30m

@@ -7354,27 +7427,26 @@ Generated query for critical alert: `max((sum(increase(src_deepsearch_question_p
-## deepsearch: deepsearch_response_truncated +## deepsearch: deepsearch_response_truncated_no_answer -

response truncated questions over 15m

+

truncated questions with no answer produced over 15m

**Descriptions** -- warning deepsearch: 3+ response truncated questions over 15m -- critical deepsearch: 10+ response truncated questions over 15m +- critical deepsearch: 3+ truncated questions with no answer produced over 15m **Next steps** -- Search frontend logs for: "answer truncated due to output token limit" to find affected questionIDs. -- This usually indicates model overthinking and spending all output tokens on thinking with no useful answer, or the user requested an extremely large output. -- Check if a recent model change may have caused increased thinking token usage. -- More help interpreting this metric is available in the [dashboards reference](dashboards#deepsearch-deepsearch_response_truncated). +- Search frontend logs for: "answer truncated due to output token limit with no response generated" to find affected questionIDs. +- The output token budget was consumed before any answer text was produced, usually by reasoning tokens, so affected users got nothing back. +- Check if a recent model change or reasoning effort change caused increased thinking token usage. +- Compare against `deepsearch_response_truncated`: elevated truncations alone are usually users requesting very large outputs and are not a regression. +- More help interpreting this metric is available in the [dashboards reference](dashboards#deepsearch-deepsearch_response_truncated_no_answer). - **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: ```json "observability.silenceAlerts": [ - "warning_deepsearch_deepsearch_response_truncated", - "critical_deepsearch_deepsearch_response_truncated" + "critical_deepsearch_deepsearch_response_truncated_no_answer" ] ``` @@ -7383,9 +7455,7 @@ Generated query for critical alert: `max((sum(increase(src_deepsearch_question_p
Technical details -Generated query for warning alert: `max((sum(increase(src_deepsearch_question_outcome_total{outcome="response_truncated"}[15m]))) > 3)` - -Generated query for critical alert: `max((sum(increase(src_deepsearch_question_outcome_total{outcome="response_truncated"}[15m]))) > 10)` +Generated query for critical alert: `max((sum(increase(src_deepsearch_question_outcome_total{outcome="response_truncated_no_answer"}[15m]))) > 3)`
diff --git a/docs/self-hosted/observability/dashboards.mdx b/docs/self-hosted/observability/dashboards.mdx index 6dc83214b..4a522b83a 100644 --- a/docs/self-hosted/observability/dashboards.mdx +++ b/docs/self-hosted/observability/dashboards.mdx @@ -15145,13 +15145,85 @@ histogram_quantile(0.95, max by (le) (rate(src_repoupdater_source_duration_secon
+#### worker: default_branch_commit_discovery_latency_p90 + +

Estimated default-branch commit discovery latency p90

+ +This panel includes only valid changes to default-branch tips. Values use Git commit timestamps and can overestimate push-to-discovery latency when commits are pushed after they are created. A missing series means no qualifying observation occurred in the query window, not zero latency. Compare update-frequency cohorts instead of aggregating them into one percentile. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101020` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +histogram_quantile(0.90, sum by (le, update_frequency) (rate(src_repoupdater_default_branch_commit_age_at_discovery_seconds_bucket[1h]))) +``` +
+ +
+ +#### worker: default_branch_commit_discovery_latency_p95 + +

Estimated default-branch commit discovery latency p95

+ +This panel includes only valid changes to default-branch tips. Values use Git commit timestamps and can overestimate push-to-discovery latency when commits are pushed after they are created. A missing series means no qualifying observation occurred in the query window, not zero latency. Compare update-frequency cohorts instead of aggregating them into one percentile. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101021` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +histogram_quantile(0.95, sum by (le, update_frequency) (rate(src_repoupdater_default_branch_commit_age_at_discovery_seconds_bucket[1h]))) +``` +
+ +
+ +#### worker: default_branch_commit_discovery_latency_p99 + +

Estimated default-branch commit discovery latency p99

+ +This panel includes only valid changes to default-branch tips. Values use Git commit timestamps and can overestimate push-to-discovery latency when commits are pushed after they are created. A missing series means no qualifying observation occurred in the query window, not zero latency. Compare update-frequency cohorts instead of aggregating them into one percentile. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101022` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +histogram_quantile(0.99, sum by (le, update_frequency) (rate(src_repoupdater_default_branch_commit_age_at_discovery_seconds_bucket[1h]))) +``` +
+ +
+ #### worker: syncer_synced_repos

Repositories synced

Refer to the [alerts reference](alerts#worker-syncer_synced_repos) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101020` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101030` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15173,7 +15245,7 @@ max(rate(src_repoupdater_syncer_synced_repos_total[1m])) Refer to the [alerts reference](alerts#worker-sourced_repos) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101021` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101031` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15195,7 +15267,7 @@ max(rate(src_repoupdater_source_repos_total[1m])) Refer to the [alerts reference](alerts#worker-sched_auto_fetch) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101030` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101040` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15220,7 +15292,7 @@ This does not indicate anything if there are no user added code hosts. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101031` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101041` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15242,7 +15314,7 @@ max(rate(src_repoupdater_sched_manual_fetch[1m])) Refer to the [alerts reference](alerts#worker-sched_loops) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101040` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101050` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15264,7 +15336,7 @@ max(rate(src_repoupdater_sched_loops[1m])) Refer to the [alerts reference](alerts#worker-src_repoupdater_stale_repos) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101041` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101051` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15286,7 +15358,7 @@ max(src_repoupdater_stale_repos) Refer to the [alerts reference](alerts#worker-sched_error) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101042` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101052` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15308,7 +15380,7 @@ max(rate(src_repoupdater_sched_error[1m])) Refer to the [alerts reference](alerts#worker-src_repoupdater_cleanup_failed_repos) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101043` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101053` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15428,6 +15500,157 @@ sum by (reason) (src_repo_statesyncer_repos_deleted{is_primary="false"})
+### Worker: Entitlements usage history telemetry: events export and queue metrics + +#### worker: entitlements_usage_history_telemetry_exported_events + +

Entitlement usage history events exported per hour

+ +The number of entitlement usage history events exported as telemetry over the last hour. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101200` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_entitlementstelemetry_usage_history_exported_events[1h])) +``` +
+ +
+ +#### worker: entitlements_usage_history_telemetry_batch_size + +

Number of entitlement usage history events exported per batch over 30m

+ +The number of entitlement usage history events exported in each batch. The largest bucket is the maximum number of events exported per batch. +If the distribution trends to the maximum bucket, export throughput is at or approaching saturation. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101201` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum by (le) (rate(src_entitlementstelemetry_usage_history_batch_size_bucket[30m])) +``` +
+ +
+ +#### worker: entitlements_usage_history_telemetry_pruned_events + +

Entitlement usage history events pruned per hour

+ +The number of already-exported entitlement usage history events pruned over the last hour. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101210` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +sum(increase(src_entitlementstelemetry_usage_history_events_pruned[1h])) +``` +
+ +
+ +#### worker: entitlements_usage_history_telemetry_queue_size + +

Entitlement usage history events pending telemetry export

+ +The number of entitlement usage history events waiting to be exported as telemetry. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101211` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +( + sum( + src_entitlementstelemetry_usage_history_queue_size + and on(instance, job) + ( + ( + increase(src_entitlementstelemetry_usage_history_queue_metrics_total{job=~"^worker.*"}[15m]) > 0 + ) + or on(instance, job) + ( + increase(src_entitlementstelemetry_usage_history_queue_metrics_errors_total{job=~"^worker.*"}[15m]) > 0 + ) + ) + ) + ) or on() vector(0) +``` +
+ +
+ +#### worker: entitlements_usage_history_telemetry_queue_oldest_age + +

Age of the oldest unexported entitlement usage history event

+ +The age in seconds of the oldest entitlement usage history event waiting to be exported as telemetry. + +Refer to the [alerts reference](alerts#worker-entitlements_usage_history_telemetry_queue_oldest_age) for 2 alerts related to this panel. + +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101220` on your Sourcegraph instance. + +*Managed by the Sourcegraph Services team.* + +
+Technical details + +Query: + +``` +( + max( + src_entitlementstelemetry_usage_history_queue_oldest_age_seconds + and on(instance, job) + ( + ( + increase(src_entitlementstelemetry_usage_history_queue_metrics_total{job=~"^worker.*"}[15m]) > 0 + ) + or on(instance, job) + ( + increase(src_entitlementstelemetry_usage_history_queue_metrics_errors_total{job=~"^worker.*"}[15m]) > 0 + ) + ) + ) + ) or on() vector(0) +``` +
+ +
+ ### Worker: External services #### worker: src_repoupdater_external_services_total @@ -15436,7 +15659,7 @@ sum by (reason) (src_repo_statesyncer_repos_deleted{is_primary="false"}) Refer to the [alerts reference](alerts#worker-src_repoupdater_external_services_total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101200` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101300` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15458,7 +15681,7 @@ max(src_repoupdater_external_services_total) Refer to the [alerts reference](alerts#worker-repoupdater_queued_sync_jobs_total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101210` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101310` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15480,7 +15703,7 @@ max(src_repoupdater_queued_sync_jobs_total) Refer to the [alerts reference](alerts#worker-repoupdater_completed_sync_jobs_total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101211` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101311` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15502,7 +15725,7 @@ max(src_repoupdater_completed_sync_jobs_total) Refer to the [alerts reference](alerts#worker-repoupdater_errored_sync_jobs_percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101212` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101312` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15524,7 +15747,7 @@ max(src_repoupdater_errored_sync_jobs_percentage) Refer to the [alerts reference](alerts#worker-github_graphql_rate_limit_remaining) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101220` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101320` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15546,7 +15769,7 @@ max by (name) (src_github_rate_limit_remaining_v2{resource="graphql"}) Refer to the [alerts reference](alerts#worker-github_rest_rate_limit_remaining) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101221` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101321` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15568,7 +15791,7 @@ max by (name) (src_github_rate_limit_remaining_v2{resource="rest"}) Refer to the [alerts reference](alerts#worker-github_search_rate_limit_remaining) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101222` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101322` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15592,7 +15815,7 @@ Indicates how long we`re waiting on the rate limit once it has been exceeded This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101230` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101330` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15616,7 +15839,7 @@ Indicates how long we`re waiting on the rate limit once it has been exceeded This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101231` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101331` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15640,7 +15863,7 @@ Indicates how long we`re waiting on the rate limit once it has been exceeded This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101232` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101332` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15662,7 +15885,7 @@ max by(name) (rate(src_github_rate_limit_wait_duration_seconds{resource="search" Refer to the [alerts reference](alerts#worker-gitlab_rest_rate_limit_remaining) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101240` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101340` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15686,7 +15909,7 @@ Indicates how long we`re waiting on the rate limit once it has been exceeded This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101241` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101341` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15710,7 +15933,7 @@ Indicates how long we`re waiting on our internal rate limiter when communicating This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101250` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101350` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15734,7 +15957,7 @@ The rate at which we fail our internal rate limiter. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101251` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101351` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15760,7 +15983,7 @@ Indicates the total number of user permissions sync completed. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101300` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101400` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15784,7 +16007,7 @@ Indicates the number of users permissions syncs completed. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101301` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101401` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15808,7 +16031,7 @@ Indicates the number of permissions syncs done for the first time for the user. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101302` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101402` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15832,7 +16055,7 @@ Indicates the total number of repo permissions sync completed. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101310` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101410` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15856,7 +16079,7 @@ Indicates the number of repos permissions syncs completed. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101311` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101411` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15880,7 +16103,7 @@ Indicates the number of permissions syncs done for the first time for the repo. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101312` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101412` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15904,7 +16127,7 @@ Indicates the max delay between two consecutive permissions sync for a user duri This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101320` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101420` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15928,7 +16151,7 @@ Indicates the max delay between two consecutive permissions sync for a repo duri This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101321` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101421` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15952,7 +16175,7 @@ Indicates the max delay between user creation and their permissions sync This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101330` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101430` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -15976,7 +16199,7 @@ Indicates the max delay between repo creation and their permissions sync This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101331` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101431` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16000,7 +16223,7 @@ Indicates the number permissions found during users/repos permissions sync. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101340` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101440` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16024,7 +16247,7 @@ Indicates the average number permissions found during permissions sync per user/ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101341` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101441` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16046,7 +16269,7 @@ avg by (type) (src_repo_perms_syncer_perms_found) Refer to the [alerts reference](alerts#worker-perms_syncer_outdated_perms) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101350` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101450` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16068,7 +16291,7 @@ max by (type) (src_repo_perms_syncer_outdated_perms) Refer to the [alerts reference](alerts#worker-perms_syncer_sync_duration) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101360` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101460` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16094,7 +16317,7 @@ Permissions sync errors are often transient and rarely actionable. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101370` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101470` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16119,7 +16342,7 @@ More about repository permissions synchronization [here](https://sourcegraph.com This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101371` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101471` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16143,7 +16366,7 @@ sum by (type) (increase(src_repo_perms_syncer_schedule_total[1m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101400` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101500` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16165,7 +16388,7 @@ sum(increase(src_gitserver_client_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101401` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101501` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16187,7 +16410,7 @@ sum by (le)(rate(src_gitserver_client_duration_seconds_bucket{job=~"^worker.*"} This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101402` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101502` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16209,7 +16432,7 @@ sum(increase(src_gitserver_client_errors_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101403` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101503` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16231,7 +16454,7 @@ sum(increase(src_gitserver_client_errors_total{job=~"^worker.*"}[5m])) / (sum(in This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101410` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101510` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16253,7 +16476,7 @@ sum by (op,scope)(increase(src_gitserver_client_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101411` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101511` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16275,7 +16498,7 @@ histogram_quantile(0.99, sum by (le,op,scope)(rate(src_gitserver_client_duratio This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101412` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101512` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16297,7 +16520,7 @@ sum by (op,scope)(increase(src_gitserver_client_errors_total{job=~"^worker.*"}[5 This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101413` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101513` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16321,7 +16544,7 @@ sum by (op,scope)(increase(src_gitserver_client_errors_total{job=~"^worker.*"}[5 This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101500` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101600` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16343,7 +16566,7 @@ sum(increase(src_gitserver_repositoryservice_client_total{job=~"^worker.*"}[5m]) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101501` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101601` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16365,7 +16588,7 @@ sum by (le)(rate(src_gitserver_repositoryservice_client_duration_seconds_bucket This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101502` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101602` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16387,7 +16610,7 @@ sum(increase(src_gitserver_repositoryservice_client_errors_total{job=~"^worker.* This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101503` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101603` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16409,7 +16632,7 @@ sum(increase(src_gitserver_repositoryservice_client_errors_total{job=~"^worker.* This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101510` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101610` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16431,7 +16654,7 @@ sum by (op,scope)(increase(src_gitserver_repositoryservice_client_total{job=~"^w This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101511` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101611` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16453,7 +16676,7 @@ histogram_quantile(0.99, sum by (le,op,scope)(rate(src_gitserver_repositoryserv This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101512` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101612` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16475,7 +16698,7 @@ sum by (op,scope)(increase(src_gitserver_repositoryservice_client_errors_total{j This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101513` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101613` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -16499,7 +16722,7 @@ sum by (op,scope)(increase(src_gitserver_repositoryservice_client_errors_total{j This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101600` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101700` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16521,7 +16744,7 @@ sum(increase(src_batches_dbstore_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101601` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101701` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16543,7 +16766,7 @@ sum by (le)(rate(src_batches_dbstore_duration_seconds_bucket{job=~"^worker.*"}[ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101602` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101702` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16565,7 +16788,7 @@ sum(increase(src_batches_dbstore_errors_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101603` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101703` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16587,7 +16810,7 @@ sum(increase(src_batches_dbstore_errors_total{job=~"^worker.*"}[5m])) / (sum(inc This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101610` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101710` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16609,7 +16832,7 @@ sum by (op)(increase(src_batches_dbstore_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101611` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101711` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16631,7 +16854,7 @@ histogram_quantile(0.99, sum by (le,op)(rate(src_batches_dbstore_duration_secon This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101612` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101712` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16653,7 +16876,7 @@ sum by (op)(increase(src_batches_dbstore_errors_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101613` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101713` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16677,7 +16900,7 @@ sum by (op)(increase(src_batches_dbstore_errors_total{job=~"^worker.*"}[5m])) / This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101700` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101800` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16699,7 +16922,7 @@ sum(increase(src_batches_service_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101701` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101801` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16721,7 +16944,7 @@ sum by (le)(rate(src_batches_service_duration_seconds_bucket{job=~"^worker.*"}[ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101702` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101802` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16743,7 +16966,7 @@ sum(increase(src_batches_service_errors_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101703` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101803` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16765,7 +16988,7 @@ sum(increase(src_batches_service_errors_total{job=~"^worker.*"}[5m])) / (sum(inc This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101710` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101810` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16787,7 +17010,7 @@ sum by (op)(increase(src_batches_service_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101711` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101811` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16809,7 +17032,7 @@ histogram_quantile(0.99, sum by (le,op)(rate(src_batches_service_duration_secon This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101712` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101812` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16831,7 +17054,7 @@ sum by (op)(increase(src_batches_service_errors_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101713` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101813` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16855,7 +17078,7 @@ sum by (op)(increase(src_batches_service_errors_total{job=~"^worker.*"}[5m])) / This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101800` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101900` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16877,7 +17100,7 @@ sum(src_query_runner_worker_processor_handlers{job=~"^worker.*"}) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101810` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101910` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16899,7 +17122,7 @@ sum(increase(src_query_runner_worker_processor_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101811` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101911` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16921,7 +17144,7 @@ sum by (le)(rate(src_query_runner_worker_processor_duration_seconds_bucket{job= This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101812` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101912` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16943,7 +17166,7 @@ sum(increase(src_query_runner_worker_processor_errors_total{job=~"^worker.*"}[5m This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101813` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101913` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16967,7 +17190,7 @@ sum(increase(src_query_runner_worker_processor_errors_total{job=~"^worker.*"}[5m This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101900` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102000` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -16989,7 +17212,7 @@ sum(increase(src_workerutil_dbworker_store_total{domain='insights_query_runner_j This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101901` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102001` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17011,7 +17234,7 @@ sum by (le)(rate(src_workerutil_dbworker_store_duration_seconds_bucket{domain=' This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101902` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102002` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17033,7 +17256,7 @@ sum(increase(src_workerutil_dbworker_store_errors_total{domain='insights_query_r This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101903` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102003` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17055,7 +17278,7 @@ sum(increase(src_workerutil_dbworker_store_errors_total{domain='insights_query_r This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101910` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102010` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17077,7 +17300,7 @@ sum by (op)(increase(src_workerutil_dbworker_store_total{domain='insights_query_ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101911` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102011` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17099,7 +17322,7 @@ histogram_quantile(0.99, sum by (le,op)(rate(src_workerutil_dbworker_store_dura This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101912` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102012` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17121,7 +17344,7 @@ sum by (op)(increase(src_workerutil_dbworker_store_errors_total{domain='insights This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=101913` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102013` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17148,7 +17371,7 @@ A value of 0 indicates the routine isn`t running currently, it awaits it`s next This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102000` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102100` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17173,7 +17396,7 @@ A low or zero value could indicate that a routine is stalled or encountering err This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102001` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102101` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17198,7 +17421,7 @@ A sustained high error rate may indicate a problem with the routine`s configurat Refer to the [alerts reference](alerts#worker-goroutine_error_rate) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102010` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102110` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -17223,7 +17446,7 @@ A value above 5% indicates that a significant portion of routine executions are Refer to the [alerts reference](alerts#worker-goroutine_error_percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102011` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102111` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -17248,7 +17471,7 @@ Longer durations might indicate increased load or processing time. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102020` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102120` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17273,7 +17496,7 @@ This represents how long a complete loop iteration takes before sleeping for the This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102021` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102121` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17298,7 +17521,7 @@ Higher values indicate that tenant processing is taking longer and may affect ov This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102030` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102130` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17323,7 +17546,7 @@ Consistently high values might indicate problematic tenants or inefficient proce This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102031` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102131` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17348,7 +17571,7 @@ Unexpected changes can indicate tenant configuration issues or scaling events. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102040` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102140` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17373,7 +17596,7 @@ A healthy routine should maintain a consistent processing rate. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102041` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102141` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17398,7 +17621,7 @@ Consistent errors indicate problems with specific tenants. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102050` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102150` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17423,7 +17646,7 @@ Values above 5% indicate significant tenant processing problems. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102051` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102151` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17447,7 +17670,7 @@ Query: This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102100` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102200` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17469,7 +17692,7 @@ sum by (app_name, db_name) (src_pgsql_conns_max_open{app_name="worker"}) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102101` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102201` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17491,7 +17714,7 @@ sum by (app_name, db_name) (src_pgsql_conns_open{app_name="worker"}) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102110` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102210` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17513,7 +17736,7 @@ sum by (app_name, db_name) (src_pgsql_conns_in_use{app_name="worker"}) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102111` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102211` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17535,7 +17758,7 @@ sum by (app_name, db_name) (src_pgsql_conns_idle{app_name="worker"}) Refer to the [alerts reference](alerts#worker-mean_blocked_seconds_per_conn_request) for 2 alerts related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102120` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102220` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17557,7 +17780,7 @@ sum by (app_name, db_name) (increase(src_pgsql_conns_blocked_seconds{app_name="w This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102130` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102230` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17579,7 +17802,7 @@ sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_idle{app_name="w This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102131` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102231` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17601,7 +17824,7 @@ sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_lifetime{app_nam This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102132` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102232` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -17625,7 +17848,7 @@ sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_idle_time{app_na Refer to the [alerts reference](alerts#worker-cpu_usage_percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102200` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102300` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17649,7 +17872,7 @@ An estimate for the active memory in use, which includes anonymous memory, file This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102201` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102301` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17673,7 +17896,7 @@ An estimate for the active memory in use in bytes, which includes anonymous memo This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102202` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102302` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17697,7 +17920,7 @@ The total anonymous memory in use by the application, which includes Go stack an Refer to the [alerts reference](alerts#worker-memory_rss) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102210` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102310` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17721,7 +17944,7 @@ This metric shows the total active file-backed memory currently in use by the ap This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102211` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102311` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17745,7 +17968,7 @@ The kernel usage metric shows the amount of memory used by the kernel on behalf This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102212` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102312` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17779,7 +18002,7 @@ value change independent of deployment events (such as an upgrade), it could ind This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102300` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102400` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17801,7 +18024,7 @@ count by(name) ((time() - container_last_seen{name=~"^worker.*"}) > 60) Refer to the [alerts reference](alerts#worker-container_cpu_usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102301` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102401` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17823,7 +18046,7 @@ cadvisor_container_cpu_usage_percentage_total{name=~"^worker.*"} Refer to the [alerts reference](alerts#worker-container_memory_usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102302` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102402` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17848,7 +18071,7 @@ When extremely high, this can indicate a resource usage problem, or can cause pr This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102303` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102403` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -17872,7 +18095,7 @@ sum by(name) (rate(container_fs_reads_total{name=~"^worker.*"}[1h]) + rate(conta Refer to the [alerts reference](alerts#worker-provisioning_container_cpu_usage_long_term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102400` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102500` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17894,7 +18117,7 @@ quantile_over_time(0.9, cadvisor_container_cpu_usage_percentage_total{name=~"^wo Refer to the [alerts reference](alerts#worker-provisioning_container_memory_usage_long_term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102401` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102501` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17916,7 +18139,7 @@ max_over_time(cadvisor_container_memory_usage_percentage_total{name=~"^worker.*" Refer to the [alerts reference](alerts#worker-provisioning_container_cpu_usage_short_term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102410` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102510` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17938,7 +18161,7 @@ max_over_time(cadvisor_container_cpu_usage_percentage_total{name=~"^worker.*"}[5 Refer to the [alerts reference](alerts#worker-provisioning_container_memory_usage_short_term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102411` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102511` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17963,7 +18186,7 @@ When it occurs frequently, it is an indicator of underprovisioning. Refer to the [alerts reference](alerts#worker-container_oomkill_events_total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102412` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102512` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -17989,7 +18212,7 @@ A high value here indicates a possible goroutine leak. Refer to the [alerts reference](alerts#worker-go_goroutines) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102500` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102600` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -18011,7 +18234,7 @@ max by(instance) (go_goroutines{job=~".*worker"}) Refer to the [alerts reference](alerts#worker-go_gc_duration_seconds) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102501` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102601` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -18035,7 +18258,7 @@ max by(instance) (go_gc_duration_seconds{job=~".*worker"}) Refer to the [alerts reference](alerts#worker-pods_available_percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102600` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102700` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -18059,7 +18282,7 @@ sum by(app) (up{app=~".*worker"}) / count by (app) (up{app=~".*worker"}) * 100 This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102700` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102800` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18081,7 +18304,7 @@ sum(increase(src_workerutil_dbworker_store_total{domain='own_background_worker_s This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102701` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102801` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18103,7 +18326,7 @@ sum by (le)(rate(src_workerutil_dbworker_store_duration_seconds_bucket{domain=' This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102702` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102802` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18125,7 +18348,7 @@ sum(increase(src_workerutil_dbworker_store_errors_total{domain='own_background_w This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102703` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102803` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18147,7 +18370,7 @@ sum(increase(src_workerutil_dbworker_store_errors_total{domain='own_background_w This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102710` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102810` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18169,7 +18392,7 @@ sum by (op)(increase(src_workerutil_dbworker_store_total{domain='own_background_ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102711` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102811` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18191,7 +18414,7 @@ histogram_quantile(0.99, sum by (le,op)(rate(src_workerutil_dbworker_store_dura This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102712` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102812` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18213,7 +18436,7 @@ sum by (op)(increase(src_workerutil_dbworker_store_errors_total{domain='own_back This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102713` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102813` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18237,7 +18460,7 @@ sum by (op)(increase(src_workerutil_dbworker_store_errors_total{domain='own_back This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102800` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102900` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -18259,7 +18482,7 @@ sum(src_own_background_worker_processor_handlers{job=~"^worker.*"}) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102810` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102910` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -18281,7 +18504,7 @@ sum(increase(src_own_background_worker_processor_total{job=~"^worker.*"}[5m])) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102811` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102911` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -18303,7 +18526,7 @@ sum by (le)(rate(src_own_background_worker_processor_duration_seconds_bucket{jo This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102812` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102912` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -18325,7 +18548,7 @@ sum(increase(src_own_background_worker_processor_errors_total{job=~"^worker.*"}[ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102813` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102913` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -18349,7 +18572,7 @@ sum(increase(src_own_background_worker_processor_errors_total{job=~"^worker.*"}[ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102900` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103000` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18371,7 +18594,7 @@ sum by (op)(increase(src_own_background_index_scheduler_total{job=~"^worker.*"}[ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102901` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103001` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18393,7 +18616,7 @@ histogram_quantile(0.99, sum by (le,op)(rate(src_own_background_index_scheduler This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102902` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103002` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18415,7 +18638,7 @@ sum by (op)(increase(src_own_background_index_scheduler_errors_total{job=~"^work This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102903` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103003` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -18441,7 +18664,7 @@ The duration since the configuration client used by the "worker" service last su This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103000` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103100` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -18463,7 +18686,7 @@ src_conf_client_time_since_last_successful_update_seconds{job=~`^worker.*`,insta Refer to the [alerts reference](alerts#worker-worker_site_configuration_duration_since_last_successful_update_by_instance) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103001` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103101` on your Sourcegraph instance. *Managed by the Sourcegraph Platform team.* @@ -26245,7 +26468,7 @@ max by (name, queue) (src_executors_managed_instance_groups_autoscaler_assigned_

Codeintel executor queue has a backlog that is not being processed

-Fires when the codeintel executor queue has a non-empty backlog (`src_workerutil_queue_depth{domain="executors_codeintel"} > 0`) while zero jobs have reached a terminal state over the last 30 minutes (`src_workerutil_dbworker_store_total{domain="codeintel_index"}`). With a backlog present, zero completions over 30m is already abnormal — the only benign zero-completion case (an empty queue) is excluded by the backlog guard. +Fires when the codeintel executor queue has a non-empty backlog (src_workerutil_queue_depth{domain="executors_codeintel"\} > 0) while zero jobs have reached a terminal state over the last 30 minutes (src_workerutil_dbworker_store_total\{domain="codeintel_index"}). With a backlog present, zero completions over 30m is already abnormal — the only benign zero-completion case (an empty queue) is excluded by the backlog guard. This is the true "executors not draining" signal: it distinguishes stuck/broken executors from executors that are simply under-provisioned. An under-provisioned queue (backlog present but jobs still completing) has a non-zero terminal-transition rate and will NOT trip this alert — that case is a capacity-review signal, not a stall. Both signals come from the worker/frontend (always scraped), not from the executor VMs, so this alert works on deployments where executors run outside the cluster (e.g. Sourcegraph Cloud). @@ -26271,7 +26494,7 @@ sum(rate(src_workerutil_dbworker_store_total{domain="codeintel_index",op=~"MarkC

Batches executor queue has a backlog that is not being processed

-Fires when the batches executor queue has a non-empty backlog (`src_workerutil_queue_depth{domain="executors_batches"} > 0`) while zero jobs have reached a terminal state over the last 30 minutes (`src_workerutil_dbworker_store_total{domain="batch_spec_workspace_execution_worker_store"}`). With a backlog present, zero completions over 30m is already abnormal — the only benign zero-completion case (an empty queue) is excluded by the backlog guard. +Fires when the batches executor queue has a non-empty backlog (src_workerutil_queue_depth{domain="executors_batches"\} > 0) while zero jobs have reached a terminal state over the last 30 minutes (src_workerutil_dbworker_store_total\{domain="batch_spec_workspace_execution_worker_store"}). With a backlog present, zero completions over 30m is already abnormal — the only benign zero-completion case (an empty queue) is excluded by the backlog guard. This is the true "executors not draining" signal: it distinguishes stuck/broken executors from executors that are simply under-provisioned. An under-provisioned queue (backlog present but jobs still completing) has a non-zero terminal-transition rate and will NOT trip this alert — that case is a capacity-review signal, not a stall. Both signals come from the worker/frontend (always scraped), not from the executor VMs, so this alert works on deployments where executors run outside the cluster (e.g. Sourcegraph Cloud). @@ -32823,6 +33046,28 @@ sum(increase(src_codeintel_uploads_janitor_scip_documents_records_altered_total{
+#### codeintel-uploads: codeintel_uploads_janitor_scip_documents_oldest_dereference_log_age + +

Approximate age of the oldest SCIP document dereference log record, based on the lowest-ID record

+ +Refer to the [alerts reference](alerts#codeintel-uploads-codeintel_uploads_janitor_scip_documents_oldest_dereference_log_age) for 1 alert related to this panel. + +To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=101302` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +(time() - max(src_codeintel_uploads_janitor_scip_documents_oldest_dereference_log_timestamp_seconds{job=~"^${source:regex}.*"})) and max(src_codeintel_uploads_janitor_scip_documents_oldest_dereference_log_timestamp_seconds{job=~"^${source:regex}.*"}) > 0 +``` +
+ +
+ #### codeintel-uploads: codeintel_uploads_janitor_scip_documents_total

Job invocation operations every 5m

@@ -34811,135 +35056,13 @@ label_replace(sum by (rpc_method, rpc_connect_rpc_error_code) (rate(rpc_client_d #### metering: metering_v1_connect_response_code_distribution -

Connect response code distribution per method over 2m

- -Rate of outbound Connect calls to enterpriseportal.metering.v1.MeteringService per method, broken down by rpc_connect_rpc_error_code. Successful calls (which carry no error-code attribute) are normalized to "ok". - -This panel has no related alerts. - -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100740` on your Sourcegraph instance. - -*Managed by the Sourcegraph Services team.* - -
-Technical details - -Query: - -``` -label_replace(sum by (rpc_method, rpc_connect_rpc_error_code) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.metering.v1.MeteringService",rpc_method=~"${metering_v1_method:regex}",job=~"${caller:regex}"}[2m])), "rpc_connect_rpc_error_code", "ok", "rpc_connect_rpc_error_code", "") -``` -
- -
- -### Metering: Broadcast Messages V1 Connect client metrics - -#### metering: broadcast_messages_v1_success_p99_duration_per_method - -

99th percentile success connect client duration per method over 2m

- -99th percentile duration of success outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. - -This panel has no related alerts. - -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100800` on your Sourcegraph instance. - -*Managed by the Sourcegraph Services team.* - -
-Technical details - -Query: - -``` -histogram_quantile(0.99, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code=""}[2m]))) -``` -
- -
- -#### metering: broadcast_messages_v1_success_p95_duration_per_method - -

95th percentile success connect client duration per method over 2m

- -95th percentile duration of success outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. - -This panel has no related alerts. - -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100801` on your Sourcegraph instance. - -*Managed by the Sourcegraph Services team.* - -
-Technical details - -Query: - -``` -histogram_quantile(0.95, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code=""}[2m]))) -``` -
- -
- -#### metering: broadcast_messages_v1_success_p90_duration_per_method - -

90th percentile success connect client duration per method over 2m

- -90th percentile duration of success outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. - -This panel has no related alerts. - -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100802` on your Sourcegraph instance. - -*Managed by the Sourcegraph Services team.* - -
-Technical details - -Query: - -``` -histogram_quantile(0.90, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code=""}[2m]))) -``` -
- -
- -#### metering: broadcast_messages_v1_failure_p99_duration_per_method - -

99th percentile failure connect client duration per method over 2m

- -99th percentile duration of failure outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. - -This panel has no related alerts. - -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100810` on your Sourcegraph instance. - -*Managed by the Sourcegraph Services team.* - -
-Technical details - -Query: - -``` -histogram_quantile(0.99, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m]))) -``` -
- -
- -#### metering: broadcast_messages_v1_failure_p95_duration_per_method - -

95th percentile failure connect client duration per method over 2m

+

Connect response code distribution per method over 2m

-95th percentile duration of failure outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. +Rate of outbound Connect calls to enterpriseportal.metering.v1.MeteringService per method, broken down by rpc_connect_rpc_error_code. Successful calls (which carry no error-code attribute) are normalized to "ok". This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100811` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100740` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -34949,21 +35072,23 @@ To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100811` Query: ``` -histogram_quantile(0.95, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m]))) +label_replace(sum by (rpc_method, rpc_connect_rpc_error_code) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.metering.v1.MeteringService",rpc_method=~"${metering_v1_method:regex}",job=~"${caller:regex}"}[2m])), "rpc_connect_rpc_error_code", "ok", "rpc_connect_rpc_error_code", "") ```
-#### metering: broadcast_messages_v1_failure_p90_duration_per_method +### Metering: Broadcast Messages V1 Connect client metrics -

90th percentile failure connect client duration per method over 2m

+#### metering: broadcast_messages_v1_success_p99_duration_per_method -90th percentile duration of failure outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. +

99th percentile success connect client duration per method over 2m

+ +99th percentile duration of success outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100812` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100800` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -34973,21 +35098,21 @@ To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100812` Query: ``` -histogram_quantile(0.90, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m]))) +histogram_quantile(0.99, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code=""}[2m]))) ```
-#### metering: broadcast_messages_v1_connect_request_rate_per_method +#### metering: broadcast_messages_v1_success_p95_duration_per_method -

Connect request rate per method over 5m

+

95th percentile success connect client duration per method over 2m

-Rate of outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method, over a 5 minute window. +95th percentile duration of success outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100820` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100801` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -34997,21 +35122,21 @@ To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100820` Query: ``` -sum by (rpc_method) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}"}[5m])) +histogram_quantile(0.95, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code=""}[2m]))) ```
-#### metering: broadcast_messages_v1_connect_error_rate_per_method +#### metering: broadcast_messages_v1_success_p90_duration_per_method -

Connect error percentage per method over 2m

+

90th percentile success connect client duration per method over 2m

-Percentage of outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService that returned a non-OK code, per method. +90th percentile duration of success outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100830` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100802` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -35021,21 +35146,21 @@ To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100830` Query: ``` -(100 * sum by (rpc_method) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m]))) / sum by (rpc_method) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}"}[2m])) +histogram_quantile(0.90, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code=""}[2m]))) ```
-#### metering: broadcast_messages_v1_connect_error_rate_per_method_by_code +#### metering: broadcast_messages_v1_failure_p99_duration_per_method -

Connect error request rate per method and code over 2m

+

99th percentile failure connect client duration per method over 2m

-Rate of failing outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, broken down by method and connect error code. +99th percentile duration of failure outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100831` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100810` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -35045,21 +35170,21 @@ To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100831` Query: ``` -label_replace(sum by (rpc_method, rpc_connect_rpc_error_code) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m])), "rpc_connect_rpc_error_code", "ok", "rpc_connect_rpc_error_code", "") +histogram_quantile(0.99, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m]))) ```
-#### metering: broadcast_messages_v1_connect_response_code_distribution +#### metering: broadcast_messages_v1_failure_p95_duration_per_method -

Connect response code distribution per method over 2m

+

95th percentile failure connect client duration per method over 2m

-Rate of outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService per method, broken down by rpc_connect_rpc_error_code. Successful calls (which carry no error-code attribute) are normalized to "ok". +95th percentile duration of failure outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100840` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100811` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -35069,29 +35194,21 @@ To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100840` Query: ``` -label_replace(sum by (rpc_method, rpc_connect_rpc_error_code) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}"}[2m])), "rpc_connect_rpc_error_code", "ok", "rpc_connect_rpc_error_code", "") +histogram_quantile(0.95, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m]))) ```
-## Entitlements usage history telemetry - -

Monitoring entitlement usage history telemetry export and queue health in Sourcegraph.

- -To see this dashboard, visit `/-/debug/grafana/d/entitlements-usage-history/entitlements-usage-history` on your Sourcegraph instance. - -### Entitlements usage history telemetry: Entitlements usage history telemetry: events export and queue metrics - -#### entitlements-usage-history: entitlements_usage_history_telemetry_exported_events +#### metering: broadcast_messages_v1_failure_p90_duration_per_method -

Entitlement usage history events exported per hour

+

90th percentile failure connect client duration per method over 2m

-The number of entitlement usage history events exported as telemetry over the last hour. +90th percentile duration of failure outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlements-usage-history?viewPanel=100000` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100812` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -35101,22 +35218,21 @@ To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlem Query: ``` -sum(increase(src_entitlementstelemetry_usage_history_exported_events[1h])) +histogram_quantile(0.90, sum by (le, rpc_method) (rate(rpc_client_duration_milliseconds_bucket{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m]))) ```
-#### entitlements-usage-history: entitlements_usage_history_telemetry_batch_size +#### metering: broadcast_messages_v1_connect_request_rate_per_method -

Number of entitlement usage history events exported per batch over 30m

+

Connect request rate per method over 5m

-The number of entitlement usage history events exported in each batch. The largest bucket is the maximum number of events exported per batch. -If the distribution trends to the maximum bucket, export throughput is at or approaching saturation. +Rate of outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, per method, over a 5 minute window. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlements-usage-history?viewPanel=100001` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100820` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -35126,21 +35242,21 @@ To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlem Query: ``` -sum by (le) (rate(src_entitlementstelemetry_usage_history_batch_size_bucket[30m])) +sum by (rpc_method) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}"}[5m])) ```
-#### entitlements-usage-history: entitlements_usage_history_telemetry_pruned_events +#### metering: broadcast_messages_v1_connect_error_rate_per_method -

Entitlement usage history events pruned per hour

+

Connect error percentage per method over 2m

-The number of already-exported entitlement usage history events pruned over the last hour. +Percentage of outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService that returned a non-OK code, per method. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlements-usage-history?viewPanel=100010` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100830` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -35150,21 +35266,21 @@ To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlem Query: ``` -sum(increase(src_entitlementstelemetry_usage_history_events_pruned[1h])) +(100 * sum by (rpc_method) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m]))) / sum by (rpc_method) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}"}[2m])) ```
-#### entitlements-usage-history: entitlements_usage_history_telemetry_queue_size +#### metering: broadcast_messages_v1_connect_error_rate_per_method_by_code -

Entitlement usage history events pending telemetry export

+

Connect error request rate per method and code over 2m

-The number of entitlement usage history events waiting to be exported as telemetry. +Rate of failing outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService, broken down by method and connect error code. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlements-usage-history?viewPanel=100011` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100831` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -35174,35 +35290,21 @@ To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlem Query: ``` -( - sum( - src_entitlementstelemetry_usage_history_queue_size - and on(instance, job) - ( - ( - increase(src_entitlementstelemetry_usage_history_queue_metrics_total{job=~"^worker.*"}[15m]) > 0 - ) - or on(instance, job) - ( - increase(src_entitlementstelemetry_usage_history_queue_metrics_errors_total{job=~"^worker.*"}[15m]) > 0 - ) - ) - ) - ) or on() vector(0) +label_replace(sum by (rpc_method, rpc_connect_rpc_error_code) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}",rpc_connect_rpc_error_code!=""}[2m])), "rpc_connect_rpc_error_code", "ok", "rpc_connect_rpc_error_code", "") ```
-#### entitlements-usage-history: entitlements_usage_history_telemetry_queue_oldest_age +#### metering: broadcast_messages_v1_connect_response_code_distribution -

Age of the oldest unexported entitlement usage history event

+

Connect response code distribution per method over 2m

-The age in seconds of the oldest entitlement usage history event waiting to be exported as telemetry. +Rate of outbound Connect calls to enterpriseportal.broadcastmessages.v1.BroadcastMessagesService per method, broken down by rpc_connect_rpc_error_code. Successful calls (which carry no error-code attribute) are normalized to "ok". This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlements-usage-history?viewPanel=100020` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/metering/metering?viewPanel=100840` on your Sourcegraph instance. *Managed by the Sourcegraph Services team.* @@ -35212,21 +35314,7 @@ To see this panel, visit `/-/debug/grafana/d/entitlements-usage-history/entitlem Query: ``` -( - max( - src_entitlementstelemetry_usage_history_queue_oldest_age_seconds - and on(instance, job) - ( - ( - increase(src_entitlementstelemetry_usage_history_queue_metrics_total{job=~"^worker.*"}[15m]) > 0 - ) - or on(instance, job) - ( - increase(src_entitlementstelemetry_usage_history_queue_metrics_errors_total{job=~"^worker.*"}[15m]) > 0 - ) - ) - ) - ) or on() vector(0) +label_replace(sum by (rpc_method, rpc_connect_rpc_error_code) (rate(rpc_client_duration_milliseconds_count{rpc_service="enterpriseportal.broadcastmessages.v1.BroadcastMessagesService",rpc_method=~"${broadcast_messages_v1_method:regex}",job=~"${caller:regex}"}[2m])), "rpc_connect_rpc_error_code", "ok", "rpc_connect_rpc_error_code", "") ``` @@ -35692,6 +35780,134 @@ sum by (code)(irate(src_http_request_duration_seconds_count{route=~"^cody.comple
+## Agent tool calls + +

Shared tool execution across MCP, Deep Search, Batch Change Agents, and other agent workloads.

+ +To see this dashboard, visit `/-/debug/grafana/d/agent-tools/agent-tools` on your Sourcegraph instance. + +### Agent tool calls: Tool calls + +#### agent-tools: agent_tool_call_rate + +

Tool calls per minute by workload, tool, and terminal status

+ +Actual executions of registered agent tools. Workload identifies the product entrypoint; status distinguishes successful calls, errors, Sourcegraph-observed timeouts, and cancellations. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/agent-tools/agent-tools?viewPanel=100000` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum by (workload, tool, status) (rate(src_agent_tool_calls_total{workload=~"$workload",tool=~"$tool"}[5m])) * 60 +``` +
+ +
+ +#### agent-tools: agent_tool_non_success_rate + +

Percentage of tool calls not completing successfully over 5m by workload and tool

+ +Share of each workload and tool`s calls ending in error, timeout, or cancellation. Compare the same tool across workloads to separate tool behavior from entrypoint-specific behavior. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/agent-tools/agent-tools?viewPanel=100001` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum by (workload, tool) (rate(src_agent_tool_calls_total{workload=~"$workload",tool=~"$tool",status!="success"}[5m])) / sum by (workload, tool) (rate(src_agent_tool_calls_total{workload=~"$workload",tool=~"$tool"}[5m])) * 100 +``` +
+ +
+ +#### agent-tools: agent_tool_p50_duration + +

Median tool call duration over 5m by workload, tool, and terminal status

+ +Typical execution time. The shared histogram has fine sub-second buckets for normally fast tools and extends through 20 minutes for intentionally slow tools. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/agent-tools/agent-tools?viewPanel=100010` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +histogram_quantile(0.50, sum by (le, workload, tool, status) (rate(src_agent_tool_call_duration_seconds_bucket{workload=~"$workload",tool=~"$tool"}[5m]))) +``` +
+ +
+ +#### agent-tools: agent_tool_p95_duration + +

95th percentile tool call duration over 5m by workload, tool, and terminal status

+ +Tail execution time. Buckets at 29s, 30s, 31s, 45s, 60s, 90s, and 120s expose common timeout boundaries, while longer buckets cover Deep Search and wait-style tools. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/agent-tools/agent-tools?viewPanel=100011` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +histogram_quantile(0.95, sum by (le, workload, tool, status) (rate(src_agent_tool_call_duration_seconds_bucket{workload=~"$workload",tool=~"$tool"}[5m]))) +``` +
+ +
+ +#### agent-tools: agent_tool_timeout_and_cancellation_count + +

Tool calls timing out or being canceled every 5m

+ +Raw timeout and cancellation counts. A sharp duration boundary in canceled calls usually indicates a client or upstream deadline; timeout means the tool returned or observed a deadline exceeded error. + +This panel has no related alerts. + +To see this panel, visit `/-/debug/grafana/d/agent-tools/agent-tools?viewPanel=100020` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum by (workload, tool, status) (increase(src_agent_tool_calls_total{workload=~"$workload",tool=~"$tool",status=~"timeout|canceled"}[5m])) +``` +
+ +
+ ## Deep Search

Monitoring for Deep Search question processing.

@@ -36016,15 +36232,39 @@ sum by (outcome) (increase(src_deepsearch_question_outcome_total{billable="true"
+#### deepsearch: deepsearch_response_truncated_no_answer + +

Truncated questions with no answer produced over 15m

+ +Number of deep search questions that hit the output token limit before producing any answer text in the last 15 minutes. Affected users received nothing, so this is the class that indicates a model or prompt regression rather than an oversized user request. + +Refer to the [alerts reference](alerts#deepsearch-deepsearch_response_truncated_no_answer) for 1 alert related to this panel. + +To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100220` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Understanding team.* + +
+Technical details + +Query: + +``` +sum(increase(src_deepsearch_question_outcome_total{outcome="response_truncated_no_answer"}[15m])) +``` +
+ +
+ #### deepsearch: deepsearch_response_truncated

Response truncated questions over 15m

-Number of deep search questions where the response was truncated due to hitting the output token limit in the last 15 minutes. +Number of deep search questions where the answer was cut off after usable text had already been produced, in the last 15 minutes. These users received an answer and can continue it with a follow-up question, so this is intentionally not alerted on: it tracks users requesting extremely large outputs. Review it on a schedule rather than in response to a page. Questions where no answer was produced at all are counted by `deepsearch_response_truncated_no_answer` instead. -Refer to the [alerts reference](alerts#deepsearch-deepsearch_response_truncated) for 2 alerts related to this panel. +This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100220` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100221` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -36048,7 +36288,7 @@ Number of deep search questions that exceeded the conversation token limit in th This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100221` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/deepsearch/deepsearch?viewPanel=100222` on your Sourcegraph instance. *Managed by the Sourcegraph Code Understanding team.* @@ -38622,4 +38862,4 @@ sum by (domain) (idelta(src_workerutil_queue_depth[10m])) / 600 ``` -
+
\ No newline at end of file