From 1cbc0487909c0f5bc2a7bda8a03677e0e87327cc Mon Sep 17 00:00:00 2001 From: Egor Ivanov Date: Fri, 18 Sep 2026 17:51:11 +0300 Subject: [PATCH 1/3] =?UTF-8?q?OpenHub:=20=D0=BD=D0=BE=D0=B2=D1=8B=D0=B9?= =?UTF-8?q?=20=D1=85=D0=B0=D0=B1=20=D0=BF=D0=B0=D0=BA=D0=B5=D1=82=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=BD=D0=B0=20hub-new.oscript.io?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - сервис openhub (segateekb/openhub:0.7.23): настройки переменными окружения, база — PostgreSQL (openhub_db), файлы пакетов — общий MinIO, бакет openhub - мониторинг хаба: otel-collector, tempo, loki, prometheus, grafana в сети monitoring; наружу смотрит только Grafana — grafana.oscript.io - сайты nginx hub-new.oscript.io и grafana.oscript.io, домены в init-letsencrypt.sh - add-letsencrypt-domain.sh — сертификат одного нового домена на работающем сервере - openhub.env.example и раздел README с порядком первого запуска Бакет openhub и учётка хаба в MinIO заводятся руками, первый администратор — мастером /setup. --- README.md | 28 + add-letsencrypt-domain.sh | 61 ++ docker-compose.yml | 136 +++ init-letsencrypt.sh | 2 +- monitoring/grafana/dashboards/openhub.json | 845 ++++++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 13 + .../provisioning/datasources/datasources.yaml | 78 ++ monitoring/loki.yaml | 51 ++ monitoring/otel-collector.yaml | 74 ++ monitoring/prometheus.yaml | 16 + monitoring/tempo.yaml | 56 ++ openhub.env.example | 11 + web/nginx/sites-enabled/grafana.oscript.io | 55 ++ web/nginx/sites-enabled/hub-new.oscript.io | 58 ++ 14 files changed, 1483 insertions(+), 1 deletion(-) create mode 100755 add-letsencrypt-domain.sh create mode 100644 monitoring/grafana/dashboards/openhub.json create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 monitoring/grafana/provisioning/datasources/datasources.yaml create mode 100644 monitoring/loki.yaml create mode 100644 monitoring/otel-collector.yaml create mode 100644 monitoring/prometheus.yaml create mode 100644 monitoring/tempo.yaml create mode 100644 openhub.env.example create mode 100644 web/nginx/sites-enabled/grafana.oscript.io create mode 100644 web/nginx/sites-enabled/hub-new.oscript.io diff --git a/README.md b/README.md index c968f52..b5cae81 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,31 @@ * Публикация в основном канале хаба, если это ветка master. Этот документ можно обсуждать и предлагать к нему правки. + +## OpenHub — hub-new.oscript.io + +Новый хаб пакетов ([OpenHub](https://github.com/Segate-ekb/openhub)) живёт в том же `docker-compose.yml`: + +| Сервис | Что это | +| --- | --- | +| `openhub` | сам хаб, образ `segateekb/openhub`; | +| `openhub_db` | PostgreSQL хаба;| +| `otel-collector`, `tempo`, `loki`, `prometheus`, `grafana` | мониторинг хаба, конфигурация в `monitoring/` | + +Файлы пакетов хаб хранит в общем MinIO + +### Первый запуск на работающем сервере + +1. Добавить в `.env` переменные из [`openhub.env.example`](openhub.env.example). +2. Завести DNS-записи `hub-new.oscript.io` и `grafana.oscript.io` на сервер. +3. Выпустить сертификаты и пересобрать nginx с новыми сайтами: + + ```bash + ./add-letsencrypt-domain.sh hub-new.oscript.io + ./add-letsencrypt-domain.sh grafana.oscript.io + ``` +4. Завести в MinIO бакет `openhub` и учётку хаба с ключами `OPENHUB_S3_ACCESS_KEY` / + `OPENHUB_S3_SECRET_KEY` из `.env` — руками, один раз. +5. Поднять хаб — база и мониторинг поднимутся сами: + +6. Сразу открыть и завести первого администратора. \ No newline at end of file diff --git a/add-letsencrypt-domain.sh b/add-letsencrypt-domain.sh new file mode 100755 index 0000000..76d1cd7 --- /dev/null +++ b/add-letsencrypt-domain.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Сертификат для одного нового домена, остальные не трогает: +# ./add-letsencrypt-domain.sh hub-new.oscript.io + +set -e + +domain="$1" +if [ -z "$domain" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +if ! [ -x "$(command -v docker-compose)" ]; then + echo 'Error: docker-compose is not installed.' >&2 + exit 1 +fi + +rsa_key_size=4096 +data_path="./web/certbot" +email="ovsiankin.aa@gmail.com" +staging=0 + +if [ -d "$data_path/conf/live/$domain" ]; then + echo "Certificate for $domain already exists, nothing to do." >&2 + exit 1 +fi + +echo "### Creating dummy certificate for $domain ..." +path="/etc/letsencrypt/live/$domain" +mkdir -p "$data_path/conf/live/$domain" +docker-compose run --rm --no-deps --entrypoint "\ + openssl req -x509 -nodes -newkey rsa:2048 -days 1 \ + -keyout '$path/privkey.pem' \ + -out '$path/fullchain.pem' \ + -subj '/CN=localhost'" certbot + +echo "### Rebuilding nginx with the new site ..." +docker-compose build nginx +docker-compose up --force-recreate --no-deps -d nginx + +echo "### Deleting dummy certificate for $domain ..." +docker-compose run --rm --no-deps --entrypoint "\ + rm -Rf /etc/letsencrypt/live/$domain && \ + rm -Rf /etc/letsencrypt/archive/$domain && \ + rm -Rf /etc/letsencrypt/renewal/$domain.conf" certbot + +echo "### Requesting Let's Encrypt certificate for $domain ..." +if [ $staging != "0" ]; then staging_arg="--staging"; fi + +docker-compose run --rm --no-deps --entrypoint "\ + certbot certonly --webroot -w /var/www/certbot \ + $staging_arg \ + --email $email \ + -d $domain \ + --rsa-key-size $rsa_key_size \ + --agree-tos \ + --non-interactive" certbot + +echo "### Reloading nginx ..." +docker-compose exec nginx nginx -s reload diff --git a/docker-compose.yml b/docker-compose.yml index 11c218b..c68f3e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,8 +109,144 @@ services: volumes: - minio_data:/data + # OpenHub — hub-new.oscript.io. Секреты — в .env, см. openhub.env.example + openhub: + image: segateekb/openhub:0.7.23 + restart: unless-stopped + environment: + - TZ=UTC + - OSHUB_INSTANCE_URL=https://hub-new.oscript.io + - OSHUB_INSTANCE_TRUSTED_PROXIES=10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 + - data_ИсточникиДанных_ТипКоннектора=КоннекторPostgreSQL + - data_ИсточникиДанных_СтрокаСоединения=Host=openhub_db;Username=openhub;Password=${OPENHUB_DB_PASSWORD};Database=openhub + - OSHUB_STORAGE_BACKEND=s3 + - OSHUB_STORAGE_S3_ENDPOINT=http://minio:9000 + - OSHUB_STORAGE_S3_BUCKET=openhub + - OSHUB_STORAGE_S3_REGION=us-east-1 + - OSHUB_STORAGE_S3_FORCE__PATH__STYLE=true + - OSHUB_STORAGE_S3_ACCESS_KEY=${OPENHUB_S3_ACCESS_KEY} + - OSHUB_STORAGE_S3_SECRET_KEY=${OPENHUB_S3_SECRET_KEY} + - Kestrel__Limits__MaxRequestBodySize=134217728 + - OSHUB_PUBLISH_MAX__UPLOAD__BYTES=134217728 + - OSHUB_PUBLISH_MAX__UNPACKED__BYTES=268435456 + - OSHUB_PUBLISH_MAX__FILES=20000 + - OTEL_ENABLED=true + - OTEL_SERVICE_NAME=openhub + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - OTEL_ENTITY_QUERY_TEXT=true + volumes: + - openhub_data:/var/lib/openhub + depends_on: + - openhub_db + - minio + - otel-collector + networks: + - default + - openhub + - monitoring + + openhub_db: + image: postgres:17-alpine + restart: unless-stopped + environment: + - POSTGRES_USER=openhub + - POSTGRES_DB=openhub + - POSTGRES_PASSWORD=${OPENHUB_DB_PASSWORD} + volumes: + - openhub_postgres_data:/var/lib/postgresql/data + networks: + - openhub + + # Мониторинг OpenHub. Наружу смотрит только Grafana: grafana.oscript.io + otel-collector: + image: otel/opentelemetry-collector-contrib:0.115.1 + command: ["--config=/etc/otelcol/config.yaml"] + restart: unless-stopped + volumes: + - ./monitoring/otel-collector.yaml:/etc/otelcol/config.yaml:ro + depends_on: + - tempo + - loki + - prometheus + networks: + - monitoring + + tempo: + image: grafana/tempo:2.7.2 + command: ["-config.file=/etc/tempo/tempo.yaml"] + restart: unless-stopped + volumes: + - ./monitoring/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + networks: + - monitoring + + loki: + image: grafana/loki:3.4.6 + command: ["-config.file=/etc/loki/loki.yaml"] + restart: unless-stopped + volumes: + - ./monitoring/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + networks: + - monitoring + + prometheus: + image: prom/prometheus:v3.5.5 + command: + - --config.file=/etc/prometheus/prometheus.yaml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --web.enable-remote-write-receiver + - --enable-feature=exemplar-storage + restart: unless-stopped + volumes: + - ./monitoring/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - prometheus_data:/prometheus + networks: + - monitoring + + grafana: + image: grafana/grafana:11.6.16 + restart: unless-stopped + environment: + - GF_SERVER_ROOT_URL=https://grafana.oscript.io + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:?задайте GRAFANA_ADMIN_PASSWORD в .env} + - GF_SECURITY_COOKIE_SECURE=true + - GF_AUTH_ANONYMOUS_ENABLED=false + - GF_USERS_ALLOW_SIGN_UP=false + - GF_USERS_DEFAULT_THEME=dark + - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/openhub.json + - GF_ANALYTICS_REPORTING_ENABLED=false + - GF_ANALYTICS_CHECK_FOR_UPDATES=false + - GF_NEWS_NEWS_FEED_ENABLED=false + volumes: + - ./monitoring/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro + - ./monitoring/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: + - prometheus + - tempo + - loki + networks: + - default + - monitoring + +networks: + openhub: + monitoring: + volumes: web_content: jenkins_home: postgres_data: minio_data: + openhub_data: + openhub_postgres_data: + tempo_data: + loki_data: + prometheus_data: + grafana_data: diff --git a/init-letsencrypt.sh b/init-letsencrypt.sh index d5b2d91..1e4d3c6 100755 --- a/init-letsencrypt.sh +++ b/init-letsencrypt.sh @@ -5,7 +5,7 @@ if ! [ -x "$(command -v docker-compose)" ]; then exit 1 fi -domains=(hub.oscript.io oscript.io build.oscript.io s3.oscript.io) +domains=(hub.oscript.io oscript.io build.oscript.io s3.oscript.io hub-new.oscript.io grafana.oscript.io) rsa_key_size=4096 data_path="./web/certbot" ssl_conf_path="./web/nginx/ssl_conf" diff --git a/monitoring/grafana/dashboards/openhub.json b/monitoring/grafana/dashboards/openhub.json new file mode 100644 index 0000000..8b11faa --- /dev/null +++ b/monitoring/grafana/dashboards/openhub.json @@ -0,0 +1,845 @@ +{ + "uid": "openhub", + "title": "OpenHub — наблюдаемость", + "description": "Что хаб делает прямо сейчас: темп запросов, задержка и ошибки по маршрутам, методы контроллеров, логи и карта сервисов. Из любой панели есть переход в трассу.", + "tags": [ + "openhub" + ], + "timezone": "browser", + "editable": true, + "schemaVersion": 39, + "version": 4, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "service", + "label": "Служба", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(traces_spanmetrics_calls_total, service)", + "refId": "СписокСлужб" + }, + "definition": "label_values(traces_spanmetrics_calls_total, service)", + "current": { + "text": "openhub", + "value": "openhub" + }, + "includeAll": false, + "multi": false, + "refresh": 2, + "sort": 1 + } + ] + }, + "panels": [ + { + "type": "row", + "id": 100, + "title": "Маршруты: темп, задержка, ошибки", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "id": 1, + "title": "Запросов в секунду", + "description": "Суммарный темп обслуженных запросов. Считается генератором метрик Tempo по серверным спанам winow.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "decimals": 2, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "graphMode": "area", + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "legendFormat": "всего", + "range": true + } + ] + }, + { + "type": "stat", + "id": 2, + "title": "Доля ошибочных запросов", + "description": "Отношение спанов со статусом Error ко всем серверным спанам. Статус Error winow ставит только на пятисотке — четырёхсотка это ответ, а не отказ.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "min": 0, + "max": 1, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "graphMode": "area", + "colorMode": "value" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(sum(rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])), 0.0000001)", + "legendFormat": "доля ошибок", + "range": true + } + ] + }, + { + "type": "timeseries", + "id": 3, + "title": "Задержка запросов: p50, p95, p99", + "description": "Квантили гистограммы traces_spanmetrics_latency. Точки-экземпляры на графике ведут в конкретную трассу.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "p50", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "p50", + "range": true + }, + { + "refId": "p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "p95", + "range": true + }, + { + "refId": "p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "p99", + "range": true + } + ] + }, + { + "type": "timeseries", + "id": 4, + "title": "Темп запросов по маршрутам", + "description": "Разбивка по имени серверного спана — «метод шаблон_маршрута», а не по конкретному адресу.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 7 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "stacking": { + "mode": "normal", + "group": "A" + }, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "legendFormat": "{{span_name}}", + "range": true + } + ] + }, + { + "type": "table", + "id": 5, + "title": "Ответы по маршрутам и кодам", + "description": "Сколько запросов на каком маршруте чем закончилось за выбранный интервал. Ошибкой спан считается только на пятисотке, поэтому четырёхсотки видны здесь, а не в доле ошибок.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Запросов", + "desc": true + } + ] + }, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "span_name": "Маршрут", + "http_response_status_code": "Код ответа", + "Value": "Запросов" + } + } + } + ], + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "format": "table", + "instant": true, + "expr": "sum by (span_name, http_response_status_code) (increase(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__range]))", + "range": false + } + ] + }, + { + "type": "table", + "id": 6, + "title": "Свежие трассы", + "description": "Поиск по Tempo. Клик по строке открывает трассу целиком; из спана внутри доступны переходы в логи и в метрики.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "options": { + "showHeader": true + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{ resource.service.name = \"$service\" }", + "limit": 20, + "tableType": "traces" + } + ] + }, + { + "type": "row", + "id": 200, + "title": "Методы контроллеров: метрики самого хаба", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "id": 7, + "title": "Задержка методов, среднее", + "description": "Среднее время вызова точки маршрута: сумма длительностей, делённая на число вызовов. Не p95: autumn-opentelemetry 1.1.0 пишет длительность в секундах, но границы корзин гистограмме не задаёт, и SDK ставит умолчательные миллисекундные ([0, 5, 10 … 10000]) — все секундные значения падают в первую корзину, и histogram_quantile вернул бы не задержку, а границу корзины. Среднее от границ не зависит; p95 вернётся, когда библиотека передаст секундные границы (issues/ЧЕРНОВИК-AUTUMN-OTEL-секунды-при-миллисекундных-границах.md). Отбор code_namespace!=\"\" оставляет метрики точек маршрута: под ту же маску имён попали гистограммы базы данных, у которых этой метки нет. Окно rate — 5 мин фиксировано: метрики хаба экспортируются раз в 60 с, и авто-интервал Grafana на коротких диапазонах пустеет.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (code_namespace, code_function_name) (rate({__name__=~\".+_duration_seconds_sum\", service_name=\"$service\", code_namespace!=\"\"}[5m])) / sum by (code_namespace, code_function_name) (rate({__name__=~\".+_duration_seconds_count\", service_name=\"$service\", code_namespace!=\"\"}[5m]))", + "legendFormat": "{{code_namespace}}.{{code_function_name}}", + "range": true + } + ] + }, + { + "type": "timeseries", + "id": 8, + "title": "Вызовы методов в секунду", + "description": "Счётчики *.counted той же точки маршрута. Метка result отделяет успех от отказа. Окно rate — 5 мин фиксировано: метрики хаба экспортируются раз в 60 с, и авто-интервал Grafana на коротких диапазонах пустеет.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (code_namespace, code_function_name, result) (rate({__name__=~\".+_counted_total\", service_name=\"$service\"}[5m]))", + "legendFormat": "{{code_namespace}}.{{code_function_name}} — {{result}}", + "range": true + } + ] + }, + { + "type": "row", + "id": 250, + "title": "База данных: операции и хранилища", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "id": 11, + "title": "Задержка обращений к СУБД, p95", + "description": "Гистограмма db.client.operation.duration наблюдателя источника данных entity (opentelemetry-instrumentation-entity). Разбивка по операции СУБД: SELECT, INSERT, COMMIT. Пусто — значит отключено otel.entity.enabled=false.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 30 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{service_name=\"$service\"}[5m])))", + "legendFormat": "{{db_operation_name}}", + "range": true + } + ] + }, + { + "type": "timeseries", + "id": 12, + "title": "Вызовы хранилищ сущностей, p95", + "description": "Гистограмма entity.repository.invocation.duration напильника слоя хранилищ. Разбивка по хранилищу и методу; метка entity_repository_state отделяет отказ от успеха. Пусто — значит отключено otel.entity.repository.enabled=false.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 30 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, entity_repository, code_function_name) (rate(entity_repository_invocation_duration_seconds_bucket{service_name=\"$service\"}[5m])))", + "legendFormat": "{{entity_repository}}.{{code_function_name}}", + "range": true + } + ] + }, + { + "type": "timeseries", + "id": 13, + "title": "Пул соединений с БД", + "description": "Датчики db.client.connection.* наблюдателя источника данных entity: занятые и свободные соединения пула, его предел (РазмерПула) и потоки, ждущие соединения. Ожидающие выше нуля — пул мал для нагрузки; занятые у предела — то же самое, только раньше.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 30 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "предел" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "db_client_connection_count{service_name=\"$service\"}", + "legendFormat": "{{db_client_connection_state}}", + "range": true + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "db_client_connection_max{service_name=\"$service\"}", + "legendFormat": "предел", + "range": true + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "db_client_connection_pending_requests{service_name=\"$service\"}", + "legendFormat": "ожидают", + "range": true + } + ] + }, + { + "type": "row", + "id": 300, + "title": "Логи и карта сервисов", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "logs", + "id": 9, + "title": "Журнал хаба", + "description": "Строки, приехавшие мостом logos → OTel. У строки с trace_id есть переход в трассу — поле trace_id живёт в structured metadata.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 0, + "y": 39 + }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{service_name=~\"$service\"}", + "queryType": "range" + } + ] + }, + { + "type": "nodeGraph", + "id": 10, + "title": "Карта сервисов", + "description": "Строится из метрик service-graph, которые генератор Tempo считает по трассам.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 39 + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "serviceMap" + } + ] + } + ] +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yaml b/monitoring/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..02bc611 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: OpenHub + orgId: 1 + folder: OpenHub + type: file + disableDeletion: false + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/monitoring/grafana/provisioning/datasources/datasources.yaml b/monitoring/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..78ed73d --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,78 @@ +# Доллар в значениях удвоен: Grafana подставляет в файл переменные окружения и одиночный $ съедает +apiVersion: 1 + +datasources: + + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + httpMethod: POST + timeInterval: 15s + prometheusType: Prometheus + prometheusVersion: 3.1.0 + exemplarTraceIdDestinations: + - name: traceID + datasourceUid: tempo + urlDisplayLabel: Открыть трассу + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: false + filterBySpanID: false + customQuery: true + query: '{service_name=~".+"} | trace_id="$${__span.traceId}"' + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: -5m + spanEndTimeShift: 5m + tags: + - key: service.name + value: service + - key: http.route + value: http_route + queries: + - name: Темп запросов маршрута + query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags, span_kind="SPAN_KIND_SERVER"}[5m]))' + - name: Задержка p95 маршрута + query: 'histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{$$__tags, span_kind="SPAN_KIND_SERVER"}[5m])) by (le))' + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: loki + search: + hide: false + streamingEnabled: + search: true + traceQuery: + timeShiftEnabled: true + spanStartTimeShift: -5m + spanEndTimeShift: 5m + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + maxLines: 1000 + derivedFields: + - name: trace_id + matcherType: label + matcherRegex: trace_id + url: '$${__value.raw}' + datasourceUid: tempo + urlDisplayLabel: Открыть трассу diff --git a/monitoring/loki.yaml b/monitoring/loki.yaml new file mode 100644 index 0000000..e736cb3 --- /dev/null +++ b/monitoring/loki.yaml @@ -0,0 +1,51 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + volume_enabled: true + reject_old_samples: false + retention_period: 168h + +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/monitoring/otel-collector.yaml b/monitoring/otel-collector.yaml new file mode 100644 index 0000000..f695491 --- /dev/null +++ b/monitoring/otel-collector.yaml @@ -0,0 +1,74 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 5s + send_batch_size: 1024 + send_batch_max_size: 2048 + + memory_limiter: + check_interval: 2s + limit_mib: 384 + spike_limit_mib: 96 + + transform/metric-resource: + error_mode: ignore + metric_statements: + - context: resource + statements: + - keep_keys(attributes, ["service.name", "service.namespace", "service.version"]) + +exporters: + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + + otlphttp/loki: + endpoint: http://loki:3100/otlp + tls: + insecure: true + + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true + add_metric_suffixes: true + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, transform/metric-resource, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + telemetry: + logs: + level: info + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 diff --git a/monitoring/prometheus.yaml b/monitoring/prometheus.yaml new file mode 100644 index 0000000..ffebdda --- /dev/null +++ b/monitoring/prometheus.yaml @@ -0,0 +1,16 @@ +storage: + exemplars: + max_exemplars: 100000 + +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] + + - job_name: otel-collector + static_configs: + - targets: ['otel-collector:8888'] diff --git a/monitoring/tempo.yaml b/monitoring/tempo.yaml new file mode 100644 index 0000000..b7096ce --- /dev/null +++ b/monitoring/tempo.yaml @@ -0,0 +1,56 @@ +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + log_level: info + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + processor: + span_metrics: + dimensions: + - http.request.method + - http.route + - http.response.status_code + histogram_buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] + service_graphs: + dimensions: + - http.request.method + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +overrides: + defaults: + metrics_generator: + processors: + - span-metrics + - service-graphs diff --git a/openhub.env.example b/openhub.env.example new file mode 100644 index 0000000..4e38103 --- /dev/null +++ b/openhub.env.example @@ -0,0 +1,11 @@ +# Добавьте в .env в корне репозитория и задайте значения на сервере. + +# Пароль базы хаба; применяется только при первом запуске. Без точки с запятой. +OPENHUB_DB_PASSWORD=change-me-strong-password + +# Учётка хаба в MinIO; её и бакет openhub заводят руками +OPENHUB_S3_ACCESS_KEY=openhub +OPENHUB_S3_SECRET_KEY=change-me-strong-secret + +# Обязателен: Grafana смотрит в интернет (grafana.oscript.io) +GRAFANA_ADMIN_PASSWORD=change-me-strong-password diff --git a/web/nginx/sites-enabled/grafana.oscript.io b/web/nginx/sites-enabled/grafana.oscript.io new file mode 100644 index 0000000..b95f1c4 --- /dev/null +++ b/web/nginx/sites-enabled/grafana.oscript.io @@ -0,0 +1,55 @@ +server { + listen 80; + listen [::]:80; + server_name grafana.oscript.io; + + access_log /var/log/nginx/access.log with_host; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name grafana.oscript.io; + + access_log /var/log/nginx/access.log with_host; + + add_header X-Content-Type-Options nosniff; + add_header X-Robots-Tag none; + add_header X-Download-Options noopen; + add_header X-Permitted-Cross-Domain-Policies none; + + add_header Strict-Transport-Security "max-age=31536000" always; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + + resolver 127.0.0.11 valid=30s; + + location / { + set $target_url http://grafana:3000; + proxy_pass $target_url; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + } + + include /etc/nginx/ssl_conf/options-ssl-nginx.conf; + ssl_dhparam /etc/nginx/ssl_conf/ssl-dhparams.pem; + ssl_certificate /etc/letsencrypt/live/grafana.oscript.io/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/grafana.oscript.io/privkey.pem; +} diff --git a/web/nginx/sites-enabled/hub-new.oscript.io b/web/nginx/sites-enabled/hub-new.oscript.io new file mode 100644 index 0000000..a7beea1 --- /dev/null +++ b/web/nginx/sites-enabled/hub-new.oscript.io @@ -0,0 +1,58 @@ +server { + listen 80; + listen [::]:80; + server_name hub-new.oscript.io; + + access_log /var/log/nginx/access.log with_host; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name hub-new.oscript.io; + + access_log /var/log/nginx/access.log with_host; + + add_header X-Content-Type-Options nosniff; + add_header X-Download-Options noopen; + add_header X-Permitted-Cross-Domain-Policies none; + + add_header Strict-Transport-Security "max-age=31536000" always; + + client_max_body_size 128M; + + proxy_redirect off; + + resolver 127.0.0.11 valid=30s; + + location / { + gzip off; + + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + + proxy_request_buffering off; + proxy_read_timeout 300; + proxy_send_timeout 300; + + set $target_url http://openhub:3333; + proxy_pass $target_url; + } + + include /etc/nginx/ssl_conf/options-ssl-nginx.conf; + ssl_dhparam /etc/nginx/ssl_conf/ssl-dhparams.pem; + ssl_certificate /etc/letsencrypt/live/hub-new.oscript.io/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/hub-new.oscript.io/privkey.pem; +} From d9b07c5d826cbca075d1ada77ee24ad33f399489 Mon Sep 17 00:00:00 2001 From: Egor Ivanov Date: Fri, 18 Sep 2026 18:24:38 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=B5=D0=B7=D0=B4?= =?UTF-8?q?=20=D0=BD=D0=B0=20lgtm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- docker-compose.yml | 89 +++---------------- ...ashboards.yaml => grafana-dashboards.yaml} | 2 +- .../provisioning/datasources/datasources.yaml | 78 ---------------- monitoring/loki.yaml | 51 ----------- .../openhub.json => openhub-dashboard.json} | 0 monitoring/otel-collector.yaml | 74 --------------- monitoring/prometheus.yaml | 16 ---- monitoring/tempo.yaml | 56 ------------ web/nginx/sites-enabled/grafana.oscript.io | 2 +- 10 files changed, 14 insertions(+), 356 deletions(-) rename monitoring/{grafana/provisioning/dashboards/dashboards.yaml => grafana-dashboards.yaml} (85%) delete mode 100644 monitoring/grafana/provisioning/datasources/datasources.yaml delete mode 100644 monitoring/loki.yaml rename monitoring/{grafana/dashboards/openhub.json => openhub-dashboard.json} (100%) delete mode 100644 monitoring/otel-collector.yaml delete mode 100644 monitoring/prometheus.yaml delete mode 100644 monitoring/tempo.yaml diff --git a/README.md b/README.md index b5cae81..3e0e350 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ | --- | --- | | `openhub` | сам хаб, образ `segateekb/openhub`; | | `openhub_db` | PostgreSQL хаба;| -| `otel-collector`, `tempo`, `loki`, `prometheus`, `grafana` | мониторинг хаба, конфигурация в `monitoring/` | +| `lgtm` | мониторинг хаба одним контейнером (`grafana/otel-lgtm`), Grafana — grafana.oscript.io; дашборд хаба — в `monitoring/` | Файлы пакетов хаб хранит в общем MinIO diff --git a/docker-compose.yml b/docker-compose.yml index c68f3e7..4a9ed5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -132,7 +132,7 @@ services: - OSHUB_PUBLISH_MAX__FILES=20000 - OTEL_ENABLED=true - OTEL_SERVICE_NAME=openhub - - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://lgtm:4318 - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf - OTEL_ENTITY_QUERY_TEXT=true volumes: @@ -140,11 +140,10 @@ services: depends_on: - openhub_db - minio - - otel-collector + - lgtm networks: - default - openhub - - monitoring openhub_db: image: postgres:17-alpine @@ -158,86 +157,23 @@ services: networks: - openhub - # Мониторинг OpenHub. Наружу смотрит только Grafana: grafana.oscript.io - otel-collector: - image: otel/opentelemetry-collector-contrib:0.115.1 - command: ["--config=/etc/otelcol/config.yaml"] - restart: unless-stopped - volumes: - - ./monitoring/otel-collector.yaml:/etc/otelcol/config.yaml:ro - depends_on: - - tempo - - loki - - prometheus - networks: - - monitoring - - tempo: - image: grafana/tempo:2.7.2 - command: ["-config.file=/etc/tempo/tempo.yaml"] - restart: unless-stopped - volumes: - - ./monitoring/tempo.yaml:/etc/tempo/tempo.yaml:ro - - tempo_data:/var/tempo - networks: - - monitoring - - loki: - image: grafana/loki:3.4.6 - command: ["-config.file=/etc/loki/loki.yaml"] - restart: unless-stopped - volumes: - - ./monitoring/loki.yaml:/etc/loki/loki.yaml:ro - - loki_data:/loki - networks: - - monitoring - - prometheus: - image: prom/prometheus:v3.5.5 - command: - - --config.file=/etc/prometheus/prometheus.yaml - - --storage.tsdb.path=/prometheus - - --storage.tsdb.retention.time=15d - - --web.enable-remote-write-receiver - - --enable-feature=exemplar-storage - restart: unless-stopped - volumes: - - ./monitoring/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro - - prometheus_data:/prometheus - networks: - - monitoring - - grafana: - image: grafana/grafana:11.6.16 + # Мониторинг OpenHub одним контейнером: коллектор OTLP, Prometheus, Tempo, Loki и Grafana. + # Наружу смотрит только Grafana: grafana.oscript.io + lgtm: + image: grafana/otel-lgtm:0.33.1 restart: unless-stopped environment: - GF_SERVER_ROOT_URL=https://grafana.oscript.io - - GF_SECURITY_ADMIN_USER=admin - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:?задайте GRAFANA_ADMIN_PASSWORD в .env} - - GF_SECURITY_COOKIE_SECURE=true + # В образе анонимный вход по умолчанию открыт с ролью Admin - GF_AUTH_ANONYMOUS_ENABLED=false - - GF_USERS_ALLOW_SIGN_UP=false - - GF_USERS_DEFAULT_THEME=dark - - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/openhub.json - - GF_ANALYTICS_REPORTING_ENABLED=false - - GF_ANALYTICS_CHECK_FOR_UPDATES=false - - GF_NEWS_NEWS_FEED_ENABLED=false volumes: - - ./monitoring/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro - - ./monitoring/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro - - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro - - grafana_data:/var/lib/grafana - depends_on: - - prometheus - - tempo - - loki - networks: - - default - - monitoring + - lgtm_data:/data + - ./monitoring/grafana-dashboards.yaml:/otel-lgtm/grafana/conf/provisioning/dashboards/openhub.yaml:ro + - ./monitoring/openhub-dashboard.json:/otel-lgtm/openhub/openhub-dashboard.json:ro networks: openhub: - monitoring: volumes: web_content: @@ -246,7 +182,4 @@ volumes: minio_data: openhub_data: openhub_postgres_data: - tempo_data: - loki_data: - prometheus_data: - grafana_data: + lgtm_data: diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yaml b/monitoring/grafana-dashboards.yaml similarity index 85% rename from monitoring/grafana/provisioning/dashboards/dashboards.yaml rename to monitoring/grafana-dashboards.yaml index 02bc611..341d0ca 100644 --- a/monitoring/grafana/provisioning/dashboards/dashboards.yaml +++ b/monitoring/grafana-dashboards.yaml @@ -9,5 +9,5 @@ providers: allowUiUpdates: false updateIntervalSeconds: 30 options: - path: /var/lib/grafana/dashboards + path: /otel-lgtm/openhub foldersFromFilesStructure: false diff --git a/monitoring/grafana/provisioning/datasources/datasources.yaml b/monitoring/grafana/provisioning/datasources/datasources.yaml deleted file mode 100644 index 78ed73d..0000000 --- a/monitoring/grafana/provisioning/datasources/datasources.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Доллар в значениях удвоен: Grafana подставляет в файл переменные окружения и одиночный $ съедает -apiVersion: 1 - -datasources: - - - name: Prometheus - uid: prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - jsonData: - httpMethod: POST - timeInterval: 15s - prometheusType: Prometheus - prometheusVersion: 3.1.0 - exemplarTraceIdDestinations: - - name: traceID - datasourceUid: tempo - urlDisplayLabel: Открыть трассу - - - name: Tempo - uid: tempo - type: tempo - access: proxy - url: http://tempo:3200 - jsonData: - tracesToLogsV2: - datasourceUid: loki - spanStartTimeShift: -5m - spanEndTimeShift: 5m - filterByTraceID: false - filterBySpanID: false - customQuery: true - query: '{service_name=~".+"} | trace_id="$${__span.traceId}"' - tracesToMetrics: - datasourceUid: prometheus - spanStartTimeShift: -5m - spanEndTimeShift: 5m - tags: - - key: service.name - value: service - - key: http.route - value: http_route - queries: - - name: Темп запросов маршрута - query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags, span_kind="SPAN_KIND_SERVER"}[5m]))' - - name: Задержка p95 маршрута - query: 'histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{$$__tags, span_kind="SPAN_KIND_SERVER"}[5m])) by (le))' - serviceMap: - datasourceUid: prometheus - nodeGraph: - enabled: true - lokiSearch: - datasourceUid: loki - search: - hide: false - streamingEnabled: - search: true - traceQuery: - timeShiftEnabled: true - spanStartTimeShift: -5m - spanEndTimeShift: 5m - - - name: Loki - uid: loki - type: loki - access: proxy - url: http://loki:3100 - jsonData: - maxLines: 1000 - derivedFields: - - name: trace_id - matcherType: label - matcherRegex: trace_id - url: '$${__value.raw}' - datasourceUid: tempo - urlDisplayLabel: Открыть трассу diff --git a/monitoring/loki.yaml b/monitoring/loki.yaml deleted file mode 100644 index e736cb3..0000000 --- a/monitoring/loki.yaml +++ /dev/null @@ -1,51 +0,0 @@ -auth_enabled: false - -server: - http_listen_port: 3100 - grpc_listen_port: 9096 - log_level: info - -common: - instance_addr: 127.0.0.1 - path_prefix: /loki - storage: - filesystem: - chunks_directory: /loki/chunks - rules_directory: /loki/rules - replication_factor: 1 - ring: - kvstore: - store: inmemory - -query_range: - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 100 - -schema_config: - configs: - - from: 2024-01-01 - store: tsdb - object_store: filesystem - schema: v13 - index: - prefix: index_ - period: 24h - -limits_config: - allow_structured_metadata: true - volume_enabled: true - reject_old_samples: false - retention_period: 168h - -compactor: - working_directory: /loki/compactor - compaction_interval: 10m - retention_enabled: true - retention_delete_delay: 2h - delete_request_store: filesystem - -analytics: - reporting_enabled: false diff --git a/monitoring/grafana/dashboards/openhub.json b/monitoring/openhub-dashboard.json similarity index 100% rename from monitoring/grafana/dashboards/openhub.json rename to monitoring/openhub-dashboard.json diff --git a/monitoring/otel-collector.yaml b/monitoring/otel-collector.yaml deleted file mode 100644 index f695491..0000000 --- a/monitoring/otel-collector.yaml +++ /dev/null @@ -1,74 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - -processors: - batch: - timeout: 5s - send_batch_size: 1024 - send_batch_max_size: 2048 - - memory_limiter: - check_interval: 2s - limit_mib: 384 - spike_limit_mib: 96 - - transform/metric-resource: - error_mode: ignore - metric_statements: - - context: resource - statements: - - keep_keys(attributes, ["service.name", "service.namespace", "service.version"]) - -exporters: - otlp/tempo: - endpoint: tempo:4317 - tls: - insecure: true - - otlphttp/loki: - endpoint: http://loki:3100/otlp - tls: - insecure: true - - prometheusremotewrite: - endpoint: http://prometheus:9090/api/v1/write - tls: - insecure: true - resource_to_telemetry_conversion: - enabled: true - add_metric_suffixes: true - -extensions: - health_check: - endpoint: 0.0.0.0:13133 - -service: - extensions: [health_check] - pipelines: - traces: - receivers: [otlp] - processors: [memory_limiter, batch] - exporters: [otlp/tempo] - metrics: - receivers: [otlp] - processors: [memory_limiter, transform/metric-resource, batch] - exporters: [prometheusremotewrite] - logs: - receivers: [otlp] - processors: [memory_limiter, batch] - exporters: [otlphttp/loki] - telemetry: - logs: - level: info - metrics: - readers: - - pull: - exporter: - prometheus: - host: 0.0.0.0 - port: 8888 diff --git a/monitoring/prometheus.yaml b/monitoring/prometheus.yaml deleted file mode 100644 index ffebdda..0000000 --- a/monitoring/prometheus.yaml +++ /dev/null @@ -1,16 +0,0 @@ -storage: - exemplars: - max_exemplars: 100000 - -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: prometheus - static_configs: - - targets: ['localhost:9090'] - - - job_name: otel-collector - static_configs: - - targets: ['otel-collector:8888'] diff --git a/monitoring/tempo.yaml b/monitoring/tempo.yaml deleted file mode 100644 index b7096ce..0000000 --- a/monitoring/tempo.yaml +++ /dev/null @@ -1,56 +0,0 @@ -stream_over_http_enabled: true - -server: - http_listen_port: 3200 - log_level: info - -distributor: - receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - -ingester: - max_block_duration: 5m - -compactor: - compaction: - block_retention: 168h - -metrics_generator: - registry: - external_labels: - source: tempo - storage: - path: /var/tempo/generator/wal - remote_write: - - url: http://prometheus:9090/api/v1/write - send_exemplars: true - processor: - span_metrics: - dimensions: - - http.request.method - - http.route - - http.response.status_code - histogram_buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] - service_graphs: - dimensions: - - http.request.method - -storage: - trace: - backend: local - wal: - path: /var/tempo/wal - local: - path: /var/tempo/blocks - -overrides: - defaults: - metrics_generator: - processors: - - span-metrics - - service-graphs diff --git a/web/nginx/sites-enabled/grafana.oscript.io b/web/nginx/sites-enabled/grafana.oscript.io index b95f1c4..c83d356 100644 --- a/web/nginx/sites-enabled/grafana.oscript.io +++ b/web/nginx/sites-enabled/grafana.oscript.io @@ -37,7 +37,7 @@ server { resolver 127.0.0.11 valid=30s; location / { - set $target_url http://grafana:3000; + set $target_url http://lgtm:3000; proxy_pass $target_url; proxy_http_version 1.1; From 2d0dfa5f4e425461422e7dbf3ee7b2af9479b5cd Mon Sep 17 00:00:00 2001 From: Egor Ivanov Date: Sat, 19 Sep 2026 22:50:00 +0300 Subject: [PATCH 3/3] =?UTF-8?q?OpenHub=200.7.24:=20=D1=83=D0=B2=D0=B5?= =?UTF-8?q?=D0=B4=D0=BE=D0=BC=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B2=20Tel?= =?UTF-8?q?egram=20=D1=85=D0=BE=D0=B4=D1=8F=D1=82=20=D1=87=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D0=B7=20=D0=BF=D1=80=D0=BE=D0=BA=D1=81=D0=B8=20=D1=85?= =?UTF-8?q?=D0=B0=D0=B1=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Прокси задаётся один раз на инсталляцию в интерфейсе: «Настройки хаба» → «Подписки» → «Доставка уведомлений» (сервер, порт, пользователь, пароль). Хранится в базе, в compose для него ничего не добавляется. Поддержан HTTP-прокси с CONNECT. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4a9ed5e..060bd3f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -111,7 +111,7 @@ services: # OpenHub — hub-new.oscript.io. Секреты — в .env, см. openhub.env.example openhub: - image: segateekb/openhub:0.7.23 + image: segateekb/openhub:0.7.24 restart: unless-stopped environment: - TZ=UTC