diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index 8d6947fea0c5..b31a96ba0f65 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -15,6 +15,12 @@ -- limitations under the License. -- +-- Nothing rendered below may depend on which plugins `plugins` or +-- `stream_plugins` in config.yaml happen to list. That list is only the +-- boot-time default: `/apisix/plugins` in etcd replaces it while APISIX runs, +-- long after this template has been rendered. A shared memory zone, or any +-- other nginx directive, cannot be added without a reload, so a plugin enabled +-- that way would otherwise find the memory it needs missing. return [=[ # Configuration File - Nginx Server Configs # This is a read-only file, do not try to modify it. @@ -65,12 +71,12 @@ env "{*name*}"; thread_pool grpc-client-nginx-module threads=1; lua { - {% if enabled_stream_plugins["prometheus"] then %} + {% if enable_stream then %} + # declared here rather than in http{} so that the stream subsystem, which + # cannot see dicts declared inside http{}, shares the same metrics lua_shared_dict prometheus-metrics {* meta.lua_shared_dict["prometheus-metrics"] *}; {% end %} - {% if enabled_plugins["prometheus"] or enabled_stream_plugins["prometheus"] then %} lua_shared_dict prometheus-cache {* meta.lua_shared_dict["prometheus-cache"] *}; - {% end %} {% if standalone_with_admin_api then %} lua_shared_dict standalone-config {* meta.lua_shared_dict["standalone-config"] *}; lua_shared_dict standalone-status {* meta.lua_shared_dict["standalone-status"] *}; @@ -82,7 +88,8 @@ lua { lua_shared_dict upstream-healthcheck {* meta.lua_shared_dict["upstream-healthcheck"] *}; } -{% if enabled_stream_plugins["prometheus"] and not enable_http then %} +{% if not enable_http and prometheus_server_addr then %} +# the stream subsystem has no server of its own to export metrics from http { lua_package_path "{*extra_lua_path*}$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;]=] .. [=[{*apisix_lua_home*}/?.lua;{*apisix_lua_home*}/?/init.lua;;{*lua_path*};"; @@ -90,8 +97,6 @@ http { .. [=[$prefix/deps/lib/lua/5.1/?.so;;]=] .. [=[{*lua_cpath*};"; - {% if enabled_stream_plugins["prometheus"] then %} - init_by_lua_block { require "resty.core" local process = require("ngx.process") @@ -125,7 +130,6 @@ http { stub_status; } } - {% end %} } {% end %} @@ -150,7 +154,7 @@ stream { # backs apisix_stream_active_connections and apisix_stream_bandwidth; the # counters live in nginx so that they keep moving during a long-lived # session instead of only being known once it ends - {% if use_apisix_base and enabled_stream_plugins["prometheus"] and stream.metrics_zone_size then %} + {% if use_apisix_base and stream.metrics_zone_size then %} apisix_stream_metrics_zone {* stream.metrics_zone_size *}; {% end %} @@ -163,9 +167,7 @@ stream { lua_shared_dict tars-stream {* stream.lua_shared_dict["tars-stream"] *}; {% end %} - {% if enabled_stream_plugins["limit-conn"] then %} lua_shared_dict plugin-limit-conn-stream {* stream.lua_shared_dict["plugin-limit-conn-stream"] *}; - {% end %} # for discovery shared dict {% if discovery_shared_dicts then %} @@ -438,79 +440,50 @@ http { lua_shared_dict plugin-ai-rate-limiting-reset-header 10m; {% end %} - {% if enabled_plugins["limit-conn"] then %} lua_shared_dict plugin-limit-conn {* http.lua_shared_dict["plugin-limit-conn"] *}; lua_shared_dict plugin-limit-conn-redis-cluster-slot-lock {* http.lua_shared_dict["plugin-limit-conn-redis-cluster-slot-lock"] *}; - {% end %} - {% if enabled_plugins["limit-req"] then %} lua_shared_dict plugin-limit-req-redis-cluster-slot-lock {* http.lua_shared_dict["plugin-limit-req-redis-cluster-slot-lock"] *}; lua_shared_dict plugin-limit-req {* http.lua_shared_dict["plugin-limit-req"] *}; - {% end %} - {% if enabled_plugins["limit-count"] then %} lua_shared_dict plugin-limit-count {* http.lua_shared_dict["plugin-limit-count"] *}; lua_shared_dict plugin-limit-count-lock {* http.lua_shared_dict["plugin-limit-count-lock"] *}; lua_shared_dict plugin-limit-count-redis-cluster-slot-lock {* http.lua_shared_dict["plugin-limit-count-redis-cluster-slot-lock"] *}; lua_shared_dict plugin-limit-count-reset-header {* http.lua_shared_dict["plugin-limit-count"] *}; - {% end %} - {% if enabled_plugins["limit-conn"] or enabled_plugins["limit-req"] or enabled_plugins["limit-count"] then %} # tracks unhealthy redis cluster nodes for fast-fail lua_shared_dict redis_cluster_health 10m; - {% end %} - {% if enabled_plugins["graphql-limit-count"] then %} lua_shared_dict plugin-graphql-limit-count {* http.lua_shared_dict["plugin-graphql-limit-count"] *}; lua_shared_dict plugin-graphql-limit-count-reset-header {* http.lua_shared_dict["plugin-graphql-limit-count-reset-header"] *}; - {% if not enabled_plugins["limit-count"] then %} - lua_shared_dict plugin-limit-count-redis-cluster-slot-lock {* http.lua_shared_dict["plugin-limit-count-redis-cluster-slot-lock"] *}; - {% end %} - {% end %} - {% if enabled_plugins["prometheus"] and not enabled_stream_plugins["prometheus"] then %} + {% if not (use_apisix_base and enable_stream) then %} + # on APISIX-Base with the stream subsystem on, this one lives in lua{} so + # that both subsystems share it lua_shared_dict prometheus-metrics {* http.lua_shared_dict["prometheus-metrics"] *}; {% end %} - {% if enabled_plugins["skywalking"] then %} lua_shared_dict tracing_buffer {* http.lua_shared_dict.tracing_buffer *}; # plugin: skywalking - {% end %} - {% if enabled_plugins["api-breaker"] then %} lua_shared_dict plugin-api-breaker {* http.lua_shared_dict["plugin-api-breaker"] *}; - {% end %} - {% if enabled_plugins["openid-connect"] or enabled_plugins["authz-keycloak"] then %} # for openid-connect and authz-keycloak plugin lua_shared_dict discovery {* http.lua_shared_dict["discovery"] *}; # cache for discovery metadata documents - {% end %} - {% if enabled_plugins["openid-connect"] then %} # for openid-connect plugin lua_shared_dict jwks {* http.lua_shared_dict["jwks"] *}; # cache for JWKs lua_shared_dict introspection {* http.lua_shared_dict["introspection"] *}; # cache for JWT verification results - {% end %} - {% if enabled_plugins["cas-auth"] then %} lua_shared_dict cas_sessions {* http.lua_shared_dict["cas-auth"] *}; - {% end %} - {% if enabled_plugins["authz-keycloak"] then %} # for authz-keycloak lua_shared_dict access-tokens {* http.lua_shared_dict["access-tokens"] *}; # cache for service account access tokens - {% end %} - {% if enabled_plugins["ocsp-stapling"] then %} lua_shared_dict ocsp-stapling {* http.lua_shared_dict["ocsp-stapling"] *}; # cache for ocsp-stapling - {% end %} - {% if enabled_plugins["ext-plugin-pre-req"] or enabled_plugins["ext-plugin-post-req"] then %} lua_shared_dict ext-plugin {* http.lua_shared_dict["ext-plugin"] *}; # cache for ext-plugin - {% end %} - {% if enabled_plugins["mcp-bridge"] or enabled_plugins["openapi-to-mcp"] then %} lua_shared_dict mcp-session {* http.lua_shared_dict["mcp-session"] *}; # cache for mcp-session - {% end %} {% if config_center == "xds" then %} lua_shared_dict xds-config 10m; @@ -524,9 +497,7 @@ http { {% end %} {% end %} - {% if enabled_plugins["error-log-logger"] then %} - lua_capture_error_log 10m; - {% end %} + lua_capture_error_log 10m; lua_ssl_verify_depth 5; ssl_session_timeout 86400; @@ -616,7 +587,7 @@ http { {% end %} } - {% if enabled_plugins["dubbo-proxy"] then %} + {% if use_apisix_base then %} upstream apisix_dubbo_backend { server 0.0.0.1; balancer_by_lua_block { @@ -710,7 +681,7 @@ http { } {% end %} - {% if enabled_plugins["prometheus"] and prometheus_server_addr then %} + {% if prometheus_server_addr then %} server { listen {* prometheus_server_addr *} reuseport; @@ -812,7 +783,7 @@ http { {% if deployment_role ~= "control_plane" then %} - {% if enabled_plugins["proxy-cache"] or enabled_plugins["graphql-proxy-cache"] then %} + {% if proxy_cache and proxy_cache.zones then %} # for proxy cache {% for _, cache in ipairs(proxy_cache.zones) do %} {% if cache.disk_path and cache.cache_levels and cache.disk_size then %} @@ -955,7 +926,7 @@ http { {% end %} # http server location configuration snippet ends - {% if enabled_plugins["dubbo-proxy"] then %} + {% if use_apisix_base then %} set $dubbo_service_name ''; set $dubbo_service_version ''; set $dubbo_method ''; @@ -1036,7 +1007,7 @@ http { # to be appended, which only $proxy_add_x_forwarded_for does. proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - {% if enabled_plugins["proxy-cache"] or enabled_plugins["graphql-proxy-cache"] then %} + {% if proxy_cache and proxy_cache.zones then %} ### the following configuration is to cache response content from upstream server set $upstream_cache_zone off; set $upstream_cache_key ''; @@ -1057,9 +1028,7 @@ http { proxy_pass $upstream_scheme://apisix_backend$upstream_uri; - {% if enabled_plugins["proxy-mirror"] then %} mirror /proxy_mirror; - {% end %} header_filter_by_lua_block { apisix.http_header_filter_phase() @@ -1096,9 +1065,7 @@ http { grpc_ssl_name $upstream_host; grpc_pass $upstream_scheme://apisix_backend; - {% if enabled_plugins["proxy-mirror"] then %} mirror /proxy_mirror_grpc; - {% end %} header_filter_by_lua_block { apisix.http_header_filter_phase() @@ -1113,7 +1080,7 @@ http { } } - {% if enabled_plugins["dubbo-proxy"] then %} + {% if use_apisix_base then %} location @dubbo_pass { access_by_lua_block { apisix.dubbo_access_phase() @@ -1137,7 +1104,6 @@ http { } {% end %} - {% if enabled_plugins["proxy-buffering"] then %} location @disable_proxy_buffering { access_by_lua_block { apisix.disable_proxy_buffering_access_phase() @@ -1160,9 +1126,7 @@ http { proxy_pass $upstream_scheme://apisix_backend$upstream_uri; - {% if enabled_plugins["proxy-mirror"] then %} mirror /proxy_mirror; - {% end %} header_filter_by_lua_block { apisix.http_header_filter_phase() @@ -1178,9 +1142,7 @@ http { proxy_buffering off; } - {% end %} - {% if enabled_plugins["proxy-mirror"] then %} location = /proxy_mirror { internal; @@ -1206,9 +1168,7 @@ http { proxy_set_header Host $upstream_host; proxy_pass $upstream_mirror_uri; } - {% end %} - {% if enabled_plugins["proxy-mirror"] then %} location = /proxy_mirror_grpc { internal; @@ -1233,7 +1193,6 @@ http { rewrite ^ $upstream_mirror_grpc_path break; grpc_pass $upstream_mirror_host; } - {% end %} } {% end %} diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index 58428421c4d1..0875f645cd7d 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -819,7 +819,7 @@ Please modify "admin_key" in conf/config.yaml . end local zipkin_set_ngx_var - if enabled_plugins["zipkin"] and yaml_conf.plugin_attr["zipkin"] then + if yaml_conf.plugin_attr["zipkin"] then zipkin_set_ngx_var = yaml_conf.plugin_attr["zipkin"].set_ngx_var end @@ -835,8 +835,6 @@ Please modify "admin_key" in conf/config.yaml . enable_http = enable_http, enable_stream = enable_stream, enabled_discoveries = enabled_discoveries, - enabled_plugins = enabled_plugins, - enabled_stream_plugins = enabled_stream_plugins, dubbo_upstream_multiplex_count = dubbo_upstream_multiplex_count, status_server_addr = status_server_addr, admin_server_addr = admin_server_addr, diff --git a/apisix/plugin.lua b/apisix/plugin.lua index 632e25c74c66..bdf4dc2c2c97 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -417,12 +417,26 @@ local function get_plugin_names(config) end +-- Whether the stream subsystem runs, which decides if the exporter has to +-- carry the L4 metrics. Deliberately not `stream_plugins` from config.yaml: +-- that list is only the boot-time default, and /apisix/plugins in etcd can +-- turn the stream prometheus plugin on later, by which time the metrics have +-- to already exist. +local function stream_subsystem_enabled() + local conf = core.config.local_conf() + local proxy_mode = conf.apisix.proxy_mode + return proxy_mode == "stream" or proxy_mode == "http&stream" +end + + function _M.load(config) local ignored, http_plugin_names, stream_plugin_names = get_plugin_names(config) if ignored then return local_plugins end + local exporter = require("apisix.plugins.prometheus.exporter") + if ngx.config.subsystem == "http" then if not http_plugin_names then core.log.error("failed to read plugin list from local file") @@ -436,6 +450,23 @@ function _M.load(config) if not ok then core.log.error("failed to load plugins: ", err) end + + -- The exporter is not built by the plugin's own init hook: that + -- hook cannot build it on a hot load, because http_init() returns + -- early outside the init phases. init_worker built it whatever + -- config.yaml listed, so here it is only restored or dropped -- + -- which is what lets /apisix/plugins turn prometheus on later. + -- + -- `active` is false on every reload, not only when prometheus was + -- just switched on: load() above unloads each plugin first, and + -- prometheus exports exporter.destroy() as its `destroy` hook. + local enabled = local_plugins_hash["prometheus"] ~= nil + local active = exporter.get_prometheus() ~= nil + if not enabled then + exporter.destroy() + elseif not active then + exporter.http_init(stream_subsystem_enabled()) + end end end @@ -922,20 +953,30 @@ end function _M.init_prometheus() - local _, http_plugin_names, stream_plugin_names = get_plugin_names() - local enabled_in_http = core.table.array_find(http_plugin_names, "prometheus") - local enabled_in_stream = core.table.array_find(stream_plugin_names, "prometheus") - + -- Not gated on the plugin list either: /apisix/plugins can enable + -- prometheus after this has run, and nothing would start the timer then. + -- It returns at once while the exporter is destroyed, which is the state + -- load() leaves it in when prometheus is not enabled. -- For stream-only mode, there are separate calls in ngx_tpl.lua. - -- And for other modes, whether in stream or http plugins, - -- the prometheus exporter needs to be initialized. - if is_http and (enabled_in_http or enabled_in_stream) then + if is_http then require("apisix.plugins.prometheus.exporter").init_exporter_timer() end end function _M.init_worker() + -- Built here rather than from the plugin's init hook, and whatever + -- `plugins` in config.yaml lists: http_init() returns early outside the + -- init phases, so it can restore what this built but cannot build it on a + -- hot load. load() below destroys it again when prometheus is not in the + -- list, so a deployment that never enables it keeps paying nothing. + local exporter = require("apisix.plugins.prometheus.exporter") + if is_http then + exporter.http_init(stream_subsystem_enabled()) + else + exporter.stream_init() + end + -- someone's plugin needs to be initialized after prometheus -- see https://github.com/apache/apisix/issues/3286 _M.load() diff --git a/apisix/plugins/prometheus.lua b/apisix/plugins/prometheus.lua index fe9e851c7b47..fc961e9e7c75 100644 --- a/apisix/plugins/prometheus.lua +++ b/apisix/plugins/prometheus.lua @@ -107,11 +107,4 @@ function _M.api() end -function _M.init() - local local_conf = core.config.local_conf() - local enabled_in_stream = core.table.array_find(local_conf.stream_plugins, "prometheus") - exporter.http_init(enabled_in_stream) -end - - return _M diff --git a/apisix/stream/plugins/prometheus.lua b/apisix/stream/plugins/prometheus.lua index 6b30ca2528ab..46222eca2472 100644 --- a/apisix/stream/plugins/prometheus.lua +++ b/apisix/stream/plugins/prometheus.lua @@ -35,8 +35,6 @@ local _M = { priority = 500, name = plugin_name, log = exporter.stream_log, - destroy = exporter.destroy, - init = exporter.stream_init, schema = schema, run_policy = "prefer_route", } diff --git a/apisix/stream/xrpc/sdk.lua b/apisix/stream/xrpc/sdk.lua index 60f100cbf8c7..d7bde3d8ae1a 100644 --- a/apisix/stream/xrpc/sdk.lua +++ b/apisix/stream/xrpc/sdk.lua @@ -22,6 +22,7 @@ local core = require("apisix.core") local config_util = require("apisix.core.config_util") local router = require("apisix.stream.router.ip_port") local metrics = require("apisix.stream.xrpc.metrics") +local plugin = require("apisix.plugin") local apisix_upstream = require("apisix.upstream") local xrpc_socket = require("resty.apisix.stream.xrpc.socket") local ngx_now = ngx.now @@ -195,6 +196,15 @@ function _M.get_metrics(session, protocol_name) if not (metric_conf and metric_conf.enable) then return nil end + + -- These are published by the prometheus stream plugin, so they are only + -- recorded while it is enabled. The effective plugin list decides that: + -- the exporter itself is now built whatever config.yaml lists, so whether + -- `metrics` happens to be initialized no longer says anything about it. + if not plugin.get_stream("prometheus") then + return nil + end + return metrics.load(protocol_name) end diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md index 95d89f02fedc..640830cdd155 100644 --- a/docs/en/latest/plugins/prometheus.md +++ b/docs/en/latest/plugins/prometheus.md @@ -784,7 +784,10 @@ The exact Upstream address and byte counts depend on the request. The active-con `apisix_stream_active_connections` and `apisix_stream_bandwidth` are backed by an NGINX shared memory zone, sized by `nginx_config.stream.metrics_zone_size` -(default `1m`). They require APISIX-Runtime; on a runtime without it the two -metrics are simply not published. +(default `1m`). The zone is allocated whenever the stream subsystem runs, not +only when `stream_plugins` in `config.yaml` lists `prometheus`, so the plugin +also works when it is enabled through `/apisix/plugins` in etcd. They require +APISIX-Runtime; on a runtime without it the two metrics are simply not +published. ::: diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md index 86081f9ffb91..da86692449e2 100644 --- a/docs/zh/latest/plugins/prometheus.md +++ b/docs/zh/latest/plugins/prometheus.md @@ -769,6 +769,6 @@ apisix_stream_bandwidth{listen_addr="0.0.0.0:9100",type="ingress",side="upstream :::note -`apisix_stream_active_connections` 和 `apisix_stream_bandwidth` 使用由 `nginx_config.stream.metrics_zone_size` 配置的 NGINX 共享内存区,默认大小为 `1m`。这两个指标依赖 APISIX-Runtime;如果运行时不提供对应模块,则不会发布这两个指标。 +`apisix_stream_active_connections` 和 `apisix_stream_bandwidth` 使用由 `nginx_config.stream.metrics_zone_size` 配置的 NGINX 共享内存区,默认大小为 `1m`。只要启用了 stream 子系统就会分配该内存区,不要求 `config.yaml` 的 `stream_plugins` 中列出 `prometheus`,因此通过 etcd 的 `/apisix/plugins` 动态启用该插件时同样有效。这两个指标依赖 APISIX-Runtime;如果运行时不提供对应模块,则不会发布这两个指标。 ::: diff --git a/t/cli/test_http_config.sh b/t/cli/test_http_config.sh index 7b8bc7e96cb6..e7d9c4f2eebf 100755 --- a/t/cli/test_http_config.sh +++ b/t/cli/test_http_config.sh @@ -64,6 +64,9 @@ echo "passed: resolve env var used as config key" git checkout conf/config.yaml +# The plugin list in config.yaml is only the boot-time default: /apisix/plugins +# in etcd replaces it while APISIX runs, and a shared dict cannot be added +# without a reload. So the dicts are rendered whatever the config file lists. echo " plugins: - ip-restriction @@ -71,21 +74,9 @@ plugins: make init -if grep "plugin-limit-conn" conf/nginx.conf > /dev/null; then - echo "failed: enable shdict on demand" - exit 1 -fi - -echo " -plugins: - - limit-conn -" > conf/config.yaml - -make init - if ! grep "plugin-limit-conn" conf/nginx.conf > /dev/null; then - echo "failed: enable shdict on demand" + echo "failed: shdict gated on the config file plugin list" exit 1 fi -echo "passed: enable shdict on demand" +echo "passed: shdict does not depend on the config file plugin list" diff --git a/t/cli/test_openapi_to_mcp.sh b/t/cli/test_openapi_to_mcp.sh index 3aef7dde80de..152a7157fc55 100755 --- a/t/cli/test_openapi_to_mcp.sh +++ b/t/cli/test_openapi_to_mcp.sh @@ -19,7 +19,10 @@ . ./t/cli/common.sh # openapi-to-mcp keeps its SSE sessions in the mcp-session shared dict, which -# is also what mcp-bridge uses. Enabling either one declares it. +# is also what mcp-bridge uses. It is declared whatever the config file plugin +# list says: that list is only the boot-time default, /apisix/plugins in etcd +# can enable either plugin later, and a shared dict cannot be added without a +# reload. echo ' plugins: @@ -40,9 +43,9 @@ plugins: make init -if grep "lua_shared_dict mcp-session" conf/nginx.conf > /dev/null; then - echo "failed: mcp-session should not be declared when no MCP plugin is enabled" +if ! grep "lua_shared_dict mcp-session" conf/nginx.conf > /dev/null; then + echo "failed: mcp-session was gated on the config file plugin list" exit 1 fi -echo "passed: openapi-to-mcp declares the mcp-session shared dict" +echo "passed: the mcp-session shared dict does not depend on the config file plugin list" diff --git a/t/cli/test_prometheus.sh b/t/cli/test_prometheus.sh index 3f0931c1802b..963e9ee258ce 100755 --- a/t/cli/test_prometheus.sh +++ b/t/cli/test_prometheus.sh @@ -185,3 +185,45 @@ fi make stop echo "passed: should use custom metric prefix" + +# The plugin list can arrive from etcd after APISIX is already running, and +# config.yaml below names no prometheus at all. The Admin API has to stay off +# for that: with it on, worker 0 overwrites /apisix/plugins from the local +# config at boot (apisix/admin/init.lua). +etcdctl del / --prefix + +echo ' +apisix: + enable_admin: false +plugins: + - ip-restriction +plugin_attr: + prometheus: + refresh_interval: 1 +' > conf/config.yaml + +make run + +# enabled only now, with APISIX already up +etcdctl put /apisix/plugins '[{"name":"prometheus"}]' + +ok=0 +deadline=$(( $(date +%s) + 20 )) +{ set +x; } 2>/dev/null +while [ "$(date +%s)" -lt "$deadline" ]; do + if curl -s --connect-timeout 1 --max-time 2 \ + http://127.0.0.1:9091/apisix/prometheus/metrics | grep -q "apisix_node_info{hostname="; then + ok=1 + break + fi + sleep 0.5 +done +set -x +if [ "$ok" -ne 1 ]; then + echo "failed: no metrics when prometheus is enabled from etcd after startup" + exit 1 +fi + +make stop + +echo "passed: prometheus can be enabled from etcd after startup" diff --git a/t/cli/test_prometheus_stream.sh b/t/cli/test_prometheus_stream.sh index 72fb96759b72..2d0b8c9a8d7f 100755 --- a/t/cli/test_prometheus_stream.sh +++ b/t/cli/test_prometheus_stream.sh @@ -124,3 +124,61 @@ if ! echo "$out" | grep "apisix_node_info{hostname=" > /dev/null; then fi echo "passed: prometheus works when only stream is enabled" + +# The plugin list can come from etcd instead of config.yaml, and it arrives long +# after nginx.conf has been rendered. The Admin API has to stay off for that: +# with it on, worker 0 overwrites /apisix/plugins from the local config at boot +# (apisix/admin/init.lua). +make stop +etcdctl del / --prefix + +echo " +apisix: + proxy_mode: http&stream + enable_admin: false + stream_proxy: + tcp: + - addr: 9100 +plugin_attr: + prometheus: + refresh_interval: 1 +" > conf/config.yaml + +make init + +if ! grep "apisix_stream_metrics_zone" conf/nginx.conf > /dev/null; then + echo "failed: the stream metrics zone was gated on the config file plugin list" + exit 1 +fi + +echo "passed: the stream metrics zone does not depend on the config file plugin list" + +etcdctl put /apisix/plugins '[{"name":"prometheus"},{"name":"prometheus","stream":true}]' +# proxied to the gateway's own HTTP port so the session carries real bytes both +# ways without a second upstream process to keep alive +etcdctl put /apisix/stream_routes/1 \ + '{"plugins":{"prometheus":{}},"upstream":{"type":"roundrobin","nodes":{"127.0.0.1:9080":1}}}' + +make run +wait_for_tcp 127.0.0.1 9100 + +# Same retry shape as the blocks above: the exporter cache populates async. +ok=0 +deadline=$(( $(date +%s) + 20 )) +{ set +x; } 2>/dev/null +while [ "$(date +%s)" -lt "$deadline" ]; do + curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9100/ >/dev/null 2>&1 || true + if curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9091/apisix/prometheus/metrics \ + | grep -qE 'apisix_stream_bandwidth\{listen_addr="0\.0\.0\.0:9100"[^}]*\} [1-9][0-9]*'; then + ok=1 + break + fi + sleep 0.5 +done +set -x +if [ "$ok" -ne 1 ]; then + echo "failed: no L4 bandwidth metric when the plugin list comes from etcd" + exit 1 +fi + +echo "passed: the L4 zone metrics work with an etcd owned plugin list" diff --git a/t/cli/test_stream_config.sh b/t/cli/test_stream_config.sh index 4eefb477eb1f..d90ebe52c849 100755 --- a/t/cli/test_stream_config.sh +++ b/t/cli/test_stream_config.sh @@ -30,34 +30,57 @@ apisix: make init +# Two, not one: the stream subsystem has no server of its own to export metrics +# from, so an http{} block is rendered to host the prometheus export server. +# That block sits inside the template's `use_apisix_base` guard, so stock +# OpenResty renders the stream block alone. This file is not guarded by +# exit_if_not_customed_nginx -- the rest of it is runtime independent -- so +# only the count is. +expected_package_path=1 +if openresty -V 2>&1 | grep apisix-nginx-module > /dev/null; then + expected_package_path=2 +fi + count=$(grep -c "lua_package_path" conf/nginx.conf) -if [ "$count" -ne 1 ]; then +if [ "$count" -ne "$expected_package_path" ]; then echo "failed: failed to enable stream proxy only by default" exit 1 fi +if grep "apisix.http_access_phase" conf/nginx.conf > /dev/null; then + echo "failed: the http proxy is enabled in stream only mode" + exit 1 +fi + echo "passed: enable stream proxy only by default" +# the export server is what puts that http{} block there, so turning it off +# takes it away again echo " apisix: enable_admin: false - proxy_mode: http&stream + proxy_mode: stream stream_proxy: tcp: - addr: 9100 +plugin_attr: + prometheus: + enable_export_server: false " > conf/config.yaml make init count=$(grep -c "lua_package_path" conf/nginx.conf) -if [ "$count" -ne 2 ]; then - echo "failed: failed to enable stream proxy and http proxy" +if [ "$count" -ne 1 ]; then + echo "failed: the prometheus export server was rendered with no export server configured" exit 1 fi +echo "passed: no http block in stream only mode without the export server" + echo " apisix: - enable_admin: true + enable_admin: false proxy_mode: http&stream stream_proxy: tcp: @@ -68,29 +91,31 @@ make init count=$(grep -c "lua_package_path" conf/nginx.conf) if [ "$count" -ne 2 ]; then - echo "failed: failed to enable stream proxy and http proxy when admin is enabled" + echo "failed: failed to enable stream proxy and http proxy" exit 1 fi -echo "passed: enable stream proxy and http proxy" - echo " apisix: + enable_admin: true proxy_mode: http&stream stream_proxy: tcp: - addr: 9100 -stream_plugins: - - ip-restriction " > conf/config.yaml make init -if grep "plugin-limit-conn-stream" conf/nginx.conf > /dev/null; then - echo "failed: enable shdict on demand" +count=$(grep -c "lua_package_path" conf/nginx.conf) +if [ "$count" -ne 2 ]; then + echo "failed: failed to enable stream proxy and http proxy when admin is enabled" exit 1 fi +echo "passed: enable stream proxy and http proxy" + +# see the same check in t/cli/test_http_config.sh: the config file plugin list +# is only the boot-time default, so nginx.conf must not depend on it echo " apisix: proxy_mode: http&stream @@ -98,14 +123,14 @@ apisix: tcp: - addr: 9100 stream_plugins: - - limit-conn + - ip-restriction " > conf/config.yaml make init if ! grep "plugin-limit-conn-stream" conf/nginx.conf > /dev/null; then - echo "failed: enable shdict on demand" + echo "failed: shdict gated on the config file plugin list" exit 1 fi -echo "passed: enable shdict on demand" +echo "passed: shdict does not depend on the config file plugin list" diff --git a/t/cli/test_zipkin_set_ngx_var.sh b/t/cli/test_zipkin_set_ngx_var.sh index 3ddd0215524f..5c68380a85e9 100755 --- a/t/cli/test_zipkin_set_ngx_var.sh +++ b/t/cli/test_zipkin_set_ngx_var.sh @@ -19,9 +19,12 @@ . ./t/cli/common.sh +# zipkin is deliberately absent from the plugin list: the variables follow +# plugin_attr alone, because /apisix/plugins in etcd can enable the plugin +# after nginx.conf has been rendered. echo ' plugins: - - zipkin + - ip-restriction plugin_attr: zipkin: set_ngx_var: true