From 7a6226ba4c4228d54e3b375789d5061c09795ed0 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 27 May 2026 11:30:58 +0300 Subject: [PATCH 01/29] pupu --- config/config.example.yaml | 2 + config/config.multicluster.yaml | 2 + internal/app/config/config.go | 418 +------------------------- internal/app/config/config_v1.go | 476 ++++++++++++++++++++++++++++++ internal/app/config/config_v2.go | 491 +++++++++++++++++++++++++++++++ internal/app/config/migrate.go | 1 + 6 files changed, 975 insertions(+), 415 deletions(-) create mode 100644 internal/app/config/config_v1.go create mode 100644 internal/app/config/config_v2.go create mode 100644 internal/app/config/migrate.go diff --git a/config/config.example.yaml b/config/config.example.yaml index 604c16c..8e0a5fa 100644 --- a/config/config.example.yaml +++ b/config/config.example.yaml @@ -1,3 +1,5 @@ +version: 2 + server: http_addr: ":5555" grpc_addr: ":5556" diff --git a/config/config.multicluster.yaml b/config/config.multicluster.yaml index 94fb312..3d1eb81 100644 --- a/config/config.multicluster.yaml +++ b/config/config.multicluster.yaml @@ -1,3 +1,5 @@ +version: 2 + server: http_addr: ":5555" grpc_addr: ":5556" diff --git a/internal/app/config/config.go b/internal/app/config/config.go index c744b57..6bea697 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -1,12 +1,9 @@ package config import ( - "bytes" "fmt" "os" "time" - - "gopkg.in/yaml.v3" ) const ( @@ -52,424 +49,15 @@ const ( defaultClickHouseReadTimeout = 30 * time.Second ) -type Config struct { - Server *Server `yaml:"server"` - Clients *Clients `yaml:"clients"` - Handlers *Handlers `yaml:"handlers"` -} - -type CORS struct { - AllowedOrigins []string `yaml:"allowed_origins"` - AllowedMethods []string `yaml:"allowed_methods"` - AllowedHeaders []string `yaml:"allowed_headers"` - ExposedHeaders []string `yaml:"exposed_headers"` - AllowCredentials bool `yaml:"allow_credentials"` - MaxAge int `yaml:"max_age"` - OptionsPassthrough bool `yaml:"options_passthrough"` -} - -type OIDC struct { - SkipVerify bool `yaml:"skip_verify"` - AuthURLs []string `yaml:"auth_urls"` - RootCA string `yaml:"root_ca"` - CACert string `yaml:"ca_cert"` - PrivateKey string `yaml:"private_key"` - SSLSkipVerify bool `yaml:"ssl_skip_verify"` - AllowedClients []string `yaml:"allowed_clients"` - CacheSecretKey string `yaml:"cache_secret_key"` -} - -type DB struct { - Name string `yaml:"name"` - Host string `yaml:"host"` - Port int64 `yaml:"port"` - Pass string `yaml:"pass"` - User string `yaml:"user"` - RequestTimeout time.Duration `yaml:"request_timeout"` - ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` - UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` -} - -func (db *DB) ConnString() string { - return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) -} - -type CH struct { - Addrs []string `yaml:"addrs"` - Database string `yaml:"database"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Sharded bool `yaml:"sharded"` - DialTimeout time.Duration `yaml:"dial_timeout"` - ReadTimeout time.Duration `yaml:"read_timeout"` -} - -type ( - RateLimiter struct { - RatePerSec int `yaml:"rate_per_sec"` - MaxBurst int `yaml:"max_burst"` - StoreMaxKeys int `yaml:"store_max_keys"` - PerHandler bool `yaml:"per_handler"` - } - - UserToRateLimiter map[string]RateLimiter - - ApiRateLimiters struct { - Default RateLimiter `yaml:"default"` - SpecialUsers UserToRateLimiter `yaml:"spec_users"` - } - - ApiToRateLimiters map[string]ApiRateLimiters -) - -type InmemoryCache struct { - NumCounters int64 `yaml:"num_counters"` - MaxCost int64 `yaml:"max_cost"` - BufferItems int64 `yaml:"buffer_items"` -} - -type Redis struct { - Addr string `yaml:"addr"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Timeout time.Duration `yaml:"timeout"` - MaxRetries int `yaml:"max_retries"` - MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` -} - -type Cache struct { - Inmemory InmemoryCache `yaml:"inmemory"` - Redis *Redis `yaml:"redis"` -} - -type S3 struct { - Endpoint string `yaml:"endpoint"` - AccessKeyID string `yaml:"access_key_id"` - SecretAccessKey string `yaml:"secret_access_key"` - BucketName string `yaml:"bucket_name"` - EnableSSl bool `yaml:"enable_ssl"` -} - -type SeqProxyDownloader struct { - Delay time.Duration `yaml:"delay"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` -} - -type SessionStore struct { - Redis Redis `yaml:"redis"` - ExportLifetime time.Duration `yaml:"export_lifetime"` -} - -type FileStore struct { - S3 *S3 `yaml:"s3"` -} - -type MassExport struct { - BatchSize uint64 `yaml:"batch_size"` - WorkersCount int `yaml:"workers_count"` - TasksChannelSize int `yaml:"tasks_channel_size"` - PartLength time.Duration `yaml:"part_length"` - URLPrefix string `yaml:"url_prefix"` - AllowedUsers []string `yaml:"allowed_users"` - FileStore *FileStore `yaml:"file_store"` - SessionStore *SessionStore `yaml:"session_store"` - SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` -} - -type Server struct { - DebugAddr string `yaml:"debug_addr"` - HTTPAddr string `yaml:"http_addr"` - GRPCAddr string `yaml:"grpc_addr"` - CORS *CORS `yaml:"cors"` - OIDC *OIDC `yaml:"oidc"` - GRPCConnectionTimeout time.Duration `yaml:"grpc_connection_timeout"` - HTTPReadHeaderTimeout time.Duration `yaml:"http_read_header_timeout"` - HTTPReadTimeout time.Duration `yaml:"http_read_timeout"` - HTTPWriteTimeout time.Duration `yaml:"http_write_timeout"` - DB *DB `yaml:"db"` - CH *CH `yaml:"clickhouse"` - RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` - Cache Cache `yaml:"cache"` - JWTSecretKey string `yaml:"jwt_secret_key"` -} - -type GRPCKeepaliveParams struct { - // After a duration of this time if the client doesn't see any activity it - // pings the server to see if the transport is still alive. - // If set below 10s, a minimum value of 10s will be used instead. - Time time.Duration `yaml:"time"` - // After having pinged for keepalive check, the client waits for a duration - // of Timeout and if no activity is seen even after that the connection is - // closed. If set below 1s, a minimum value of 1s will be used instead. - Timeout time.Duration `yaml:"timeout"` - // If true, client sends keepalive pings even with no active RPCs. If false, - // when there are no active RPCs, Time and Timeout will be ignored and no - // keepalive pings will be sent. False by default. - PermitWithoutStream bool `yaml:"permit_without_stream"` -} - -type SeqDBClient struct { - ID string `yaml:"id"` - Timeout time.Duration `yaml:"timeout"` - AvgDocSize int `yaml:"avg_doc_size"` - Addrs []string `yaml:"addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ClientMode string `yaml:"client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` -} - -type Clients struct { - SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` - SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` - SeqDBAddrs []string `yaml:"seq_db_addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ProxyClientMode string `yaml:"proxy_client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` - SeqDB []SeqDBClient `yaml:"seq_db"` -} - -type Handlers struct { - SeqAPI SeqAPI `yaml:"seq_api"` - ErrorGroups ErrorGroups `yaml:"error_groups"` - MassExport *MassExport `yaml:"mass_export"` - AsyncSearch AsyncSearch `yaml:"async_search"` -} - -type Field struct { - Name string `yaml:"name"` - Type string `yaml:"type"` -} - -type SeqAPI struct { - *SeqAPIOptions `yaml:",inline"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` -} - -type SeqAPIEnv struct { - SeqDB string `yaml:"seq_db_id"` - Options *SeqAPIOptions `yaml:"options"` -} - -type SeqAPIOptions struct { - MaxSearchLimit int32 `yaml:"max_search_limit"` - MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` - MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` - MaxExportLimit int32 `yaml:"max_export_limit"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` - EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` - PinnedFields []Field `yaml:"pinned_fields"` - SystemFields []Field `yaml:"system_fields"` - LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` - LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` - FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` - Masking *Masking `yaml:"masking"` -} - -type Masking struct { - Masks []Mask `yaml:"masks"` - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` -} - -type Mask struct { - Re string `yaml:"re"` - Groups []int `yaml:"groups"` - Mode string `yaml:"mode"` // "mask" or "replace" or "cut" - ReplaceWord string `yaml:"replace_word"` // for mode:replace - - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` - - FieldFilters *FieldFilterSet `yaml:"field_filters"` -} - -type FieldFilter struct { - Field string `yaml:"field"` - Mode string `yaml:"mode"` // "equal" or "contains" or "prefix" or "suffix" - Values []string `yaml:"values"` -} - -type FieldFilterSet struct { - Condition string `yaml:"condition"` // "and" or "or" or "not" - Filters []FieldFilter `yaml:"filters"` // max 1 if condition:not -} - -type LogTagsMapping struct { - Release []string `yaml:"release"` - Env []string `yaml:"env"` -} - -type ErrorGroups struct { - LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` - QueryFilter map[string]string `yaml:"query_filter"` -} - -type AsyncSearch struct { - AdminUsers []string `yaml:"admin_users"` - ListQueryLengthLimit int `yaml:"list_query_length_limit"` -} - -// FromFile parse config from config path. func FromFile(cfgPath string) (Config, error) { - cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec + cfgBytes, err := os.ReadFile(cfgPath) if err != nil { return Config{}, fmt.Errorf("error reading file: %s", err) } - cfg, err := parse(cfgBytes) - if err != nil { - return Config{}, fmt.Errorf("error parsing file: %s", err) - } - - proxyClientMode := cfg.Clients.ProxyClientMode - if proxyClientMode == "" { - cfg.Clients.ProxyClientMode = ProxyClientModeGRPC - } else if proxyClientMode != ProxyClientModeGRPC { - return Config{}, fmt.Errorf( - "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", - proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, - ) - } - - setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) - - if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { - cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit - } - - if cfg.Server.DB != nil && cfg.Server.DB.UsePreparedStatements == nil { - cfg.Server.DB.UsePreparedStatements = new(bool) - *cfg.Server.DB.UsePreparedStatements = true - } - - if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { - cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout - } - - if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { - cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout - } - - if cfg.Clients.GRPCKeepaliveParams != nil { - if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { - cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime - } - if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { - cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout - } - } - - if cfg.Server.Cache.Inmemory.NumCounters <= 0 { - cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters - } - if cfg.Server.Cache.Inmemory.MaxCost <= 0 { - cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost - } - if cfg.Server.Cache.Inmemory.BufferItems <= 0 { - cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems - } - - if len(cfg.Clients.SeqDB) == 0 { - defaultClient := SeqDBClient{ - ID: DefaultSeqDBClientID, - Timeout: cfg.Clients.SeqDBTimeout, - AvgDocSize: cfg.Clients.SeqDBAvgDocSize, - Addrs: cfg.Clients.SeqDBAddrs, - RequestRetries: cfg.Clients.RequestRetries, - InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, - MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, - ClientMode: cfg.Clients.ProxyClientMode, - GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, - } - cfg.Clients.SeqDB = []SeqDBClient{defaultClient} - } - - clientIDs := make(map[string]struct{}) - for _, client := range cfg.Clients.SeqDB { - if client.ID == "" { - return Config{}, fmt.Errorf("seq_db client ID cannot be empty") - } - if _, ok := clientIDs[client.ID]; ok { - return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) - } - clientIDs[client.ID] = struct{}{} - } - - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return Config{}, fmt.Errorf("default_env must be specified when using envs") - } - - if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) - } - - for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if _, ok := clientIDs[envConfig.SeqDB]; !ok { - return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) - } - - if envConfig.Options == nil { - envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions - } else { - setSeqAPIOptionsDefaults(envConfig.Options) - } - - cfg.Handlers.SeqAPI.Envs[envName] = envConfig - } - } - - return cfg, nil -} - -func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { - if options.MaxAggregationsPerRequest <= 0 { - options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest - } - if options.MaxBucketsPerAggregationTs <= 0 { - options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs - } - if options.MaxParallelExportRequests <= 0 { - options.MaxParallelExportRequests = defaultMaxParallelExportRequests - } - if options.MaxSearchTotalLimit <= 0 { - options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit - } - if options.MaxSearchOffsetLimit <= 0 { - options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit - } - if options.MaxExportLimit <= 0 { - options.MaxExportLimit = defaultMaxExportLimit - } - if options.EventsCacheTTL <= 0 { - options.EventsCacheTTL = defaultEventsCacheTTL - } - if options.LogsLifespanCacheKey == "" { - options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey - } - if options.LogsLifespanCacheTTL <= 0 { - options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL - } + return fromBytes(cfgBytes) } -func parse(cfg []byte) (Config, error) { - result := Config{} - - decoder := yaml.NewDecoder(bytes.NewReader(cfg)) - decoder.KnownFields(true) - if err := decoder.Decode(&result); err != nil { - return result, fmt.Errorf("error parsing config: %w", err) - } +func fromBytes(cfgBytes []byte) (Config, error) { - return result, nil } diff --git a/internal/app/config/config_v1.go b/internal/app/config/config_v1.go new file mode 100644 index 0000000..589e661 --- /dev/null +++ b/internal/app/config/config_v1.go @@ -0,0 +1,476 @@ +package config + +import ( + "bytes" + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + DefaultSeqDBClientID = "default" + + ProxyClientModeGRPC = "grpc" + + MaskModeMask = "mask" + MaskModeReplace = "replace" + MaskModeCut = "cut" + + FieldFilterConditionAnd = "and" + FieldFilterConditionOr = "or" + FieldFilterConditionNot = "not" + + FieldFilterModeEqual = "equal" + FieldFilterModeContains = "contains" + FieldFilterModePrefix = "prefix" + FieldFilterModeSuffix = "suffix" + + minGRPCKeepaliveTime = 10 * time.Second + minGRPCKeepaliveTimeout = 1 * time.Second + + defaultAsyncSearchListQueryLengthLimit = 1000 + + defaultMaxSearchTotalLimit = 1000000 + defaultMaxSearchOffsetLimit = 1000000 + defaultMaxExportLimit = 100000 + defaultMaxAggregationsPerRequest = 1 + defaultMaxBucketsPerAggregationTs = 200 + defaultMaxParallelExportRequests = 1 + + defaultInmemCacheNumCounters = 10000000 + defaultInmemCacheMaxCost = 1000000 + defaultInmemCacheBufferItems = 64 + + defaultEventsCacheTTL = 24 * time.Hour + + defaultLogsLifespanCacheKey = "logs_lifespan" + defaultLogsLifespanCacheTTL = 10 * time.Minute + + defaultClickHouseDialTimeout = 5 * time.Second + defaultClickHouseReadTimeout = 30 * time.Second +) + +type Config struct { + Version *int `yaml:"version,omitempty"` + Server *Server `yaml:"server"` + Clients *Clients `yaml:"clients"` + Handlers *Handlers `yaml:"handlers"` +} + +type CORS struct { + AllowedOrigins []string `yaml:"allowed_origins"` + AllowedMethods []string `yaml:"allowed_methods"` + AllowedHeaders []string `yaml:"allowed_headers"` + ExposedHeaders []string `yaml:"exposed_headers"` + AllowCredentials bool `yaml:"allow_credentials"` + MaxAge int `yaml:"max_age"` + OptionsPassthrough bool `yaml:"options_passthrough"` +} + +type OIDC struct { + SkipVerify bool `yaml:"skip_verify"` + AuthURLs []string `yaml:"auth_urls"` + RootCA string `yaml:"root_ca"` + CACert string `yaml:"ca_cert"` + PrivateKey string `yaml:"private_key"` + SSLSkipVerify bool `yaml:"ssl_skip_verify"` + AllowedClients []string `yaml:"allowed_clients"` + CacheSecretKey string `yaml:"cache_secret_key"` +} + +type DB struct { + Name string `yaml:"name"` + Host string `yaml:"host"` + Port int64 `yaml:"port"` + Pass string `yaml:"pass"` + User string `yaml:"user"` + RequestTimeout time.Duration `yaml:"request_timeout"` + ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` + UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` +} + +func (db *DB) ConnString() string { + return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) +} + +type CH struct { + Addrs []string `yaml:"addrs"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Sharded bool `yaml:"sharded"` + DialTimeout time.Duration `yaml:"dial_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` +} + +type ( + RateLimiter struct { + RatePerSec int `yaml:"rate_per_sec"` + MaxBurst int `yaml:"max_burst"` + StoreMaxKeys int `yaml:"store_max_keys"` + PerHandler bool `yaml:"per_handler"` + } + + UserToRateLimiter map[string]RateLimiter + + ApiRateLimiters struct { + Default RateLimiter `yaml:"default"` + SpecialUsers UserToRateLimiter `yaml:"spec_users"` + } + + ApiToRateLimiters map[string]ApiRateLimiters +) + +type InmemoryCache struct { + NumCounters int64 `yaml:"num_counters"` + MaxCost int64 `yaml:"max_cost"` + BufferItems int64 `yaml:"buffer_items"` +} + +type Redis struct { + Addr string `yaml:"addr"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Timeout time.Duration `yaml:"timeout"` + MaxRetries int `yaml:"max_retries"` + MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` +} + +type Cache struct { + Inmemory InmemoryCache `yaml:"inmemory"` + Redis *Redis `yaml:"redis"` +} + +type S3 struct { + Endpoint string `yaml:"endpoint"` + AccessKeyID string `yaml:"access_key_id"` + SecretAccessKey string `yaml:"secret_access_key"` + BucketName string `yaml:"bucket_name"` + EnableSSl bool `yaml:"enable_ssl"` +} + +type SeqProxyDownloader struct { + Delay time.Duration `yaml:"delay"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` +} + +type SessionStore struct { + Redis Redis `yaml:"redis"` + ExportLifetime time.Duration `yaml:"export_lifetime"` +} + +type FileStore struct { + S3 *S3 `yaml:"s3"` +} + +type MassExport struct { + BatchSize uint64 `yaml:"batch_size"` + WorkersCount int `yaml:"workers_count"` + TasksChannelSize int `yaml:"tasks_channel_size"` + PartLength time.Duration `yaml:"part_length"` + URLPrefix string `yaml:"url_prefix"` + AllowedUsers []string `yaml:"allowed_users"` + FileStore *FileStore `yaml:"file_store"` + SessionStore *SessionStore `yaml:"session_store"` + SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` +} + +type Server struct { + DebugAddr string `yaml:"debug_addr"` + HTTPAddr string `yaml:"http_addr"` + GRPCAddr string `yaml:"grpc_addr"` + CORS *CORS `yaml:"cors"` + OIDC *OIDC `yaml:"oidc"` + GRPCConnectionTimeout time.Duration `yaml:"grpc_connection_timeout"` + HTTPReadHeaderTimeout time.Duration `yaml:"http_read_header_timeout"` + HTTPReadTimeout time.Duration `yaml:"http_read_timeout"` + HTTPWriteTimeout time.Duration `yaml:"http_write_timeout"` + DB *DB `yaml:"db"` + CH *CH `yaml:"clickhouse"` + RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` + Cache Cache `yaml:"cache"` + JWTSecretKey string `yaml:"jwt_secret_key"` +} + +type GRPCKeepaliveParams struct { + // After a duration of this time if the client doesn't see any activity it + // pings the server to see if the transport is still alive. + // If set below 10s, a minimum value of 10s will be used instead. + Time time.Duration `yaml:"time"` + // After having pinged for keepalive check, the client waits for a duration + // of Timeout and if no activity is seen even after that the connection is + // closed. If set below 1s, a minimum value of 1s will be used instead. + Timeout time.Duration `yaml:"timeout"` + // If true, client sends keepalive pings even with no active RPCs. If false, + // when there are no active RPCs, Time and Timeout will be ignored and no + // keepalive pings will be sent. False by default. + PermitWithoutStream bool `yaml:"permit_without_stream"` +} + +type SeqDBClient struct { + ID string `yaml:"id"` + Timeout time.Duration `yaml:"timeout"` + AvgDocSize int `yaml:"avg_doc_size"` + Addrs []string `yaml:"addrs"` + RequestRetries int `yaml:"request_retries"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + ClientMode string `yaml:"client_mode"` + GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` +} + +type Clients struct { + SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` + SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` + SeqDBAddrs []string `yaml:"seq_db_addrs"` + RequestRetries int `yaml:"request_retries"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + ProxyClientMode string `yaml:"proxy_client_mode"` + GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` + SeqDB []SeqDBClient `yaml:"seq_db"` +} + +type Handlers struct { + SeqAPI SeqAPI `yaml:"seq_api"` + ErrorGroups ErrorGroups `yaml:"error_groups"` + MassExport *MassExport `yaml:"mass_export"` + AsyncSearch AsyncSearch `yaml:"async_search"` +} + +type Field struct { + Name string `yaml:"name"` + Type string `yaml:"type"` +} + +type SeqAPI struct { + *SeqAPIOptions `yaml:",inline"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type SeqAPIEnv struct { + SeqDB string `yaml:"seq_db_id"` + Options *SeqAPIOptions `yaml:"options"` +} + +type SeqAPIOptions struct { + MaxSearchLimit int32 `yaml:"max_search_limit"` + MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` + MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` + MaxExportLimit int32 `yaml:"max_export_limit"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` + EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` + PinnedFields []Field `yaml:"pinned_fields"` + SystemFields []Field `yaml:"system_fields"` + LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` + LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` + FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` + Masking *Masking `yaml:"masking"` +} + +type Masking struct { + Masks []Mask `yaml:"masks"` + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` +} + +type Mask struct { + Re string `yaml:"re"` + Groups []int `yaml:"groups"` + Mode string `yaml:"mode"` // "mask" or "replace" or "cut" + ReplaceWord string `yaml:"replace_word"` // for mode:replace + + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` + + FieldFilters *FieldFilterSet `yaml:"field_filters"` +} + +type FieldFilter struct { + Field string `yaml:"field"` + Mode string `yaml:"mode"` // "equal" or "contains" or "prefix" or "suffix" + Values []string `yaml:"values"` +} + +type FieldFilterSet struct { + Condition string `yaml:"condition"` // "and" or "or" or "not" + Filters []FieldFilter `yaml:"filters"` // max 1 if condition:not +} + +type LogTagsMapping struct { + Release []string `yaml:"release"` + Env []string `yaml:"env"` +} + +type ErrorGroups struct { + LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` + QueryFilter map[string]string `yaml:"query_filter"` +} + +type AsyncSearch struct { + AdminUsers []string `yaml:"admin_users"` + ListQueryLengthLimit int `yaml:"list_query_length_limit"` +} + +// FromFile parse config from config path. +func FromFile(cfgPath string) (Config, error) { + cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec + if err != nil { + return Config{}, fmt.Errorf("error reading file: %s", err) + } + + cfg, err := parse(cfgBytes) + if err != nil { + return Config{}, fmt.Errorf("error parsing file: %s", err) + } + + proxyClientMode := cfg.Clients.ProxyClientMode + if proxyClientMode == "" { + cfg.Clients.ProxyClientMode = ProxyClientModeGRPC + } else if proxyClientMode != ProxyClientModeGRPC { + return Config{}, fmt.Errorf( + "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", + proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, + ) + } + + setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + + if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { + cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + } + + if cfg.Server.DB != nil && cfg.Server.DB.UsePreparedStatements == nil { + cfg.Server.DB.UsePreparedStatements = new(bool) + *cfg.Server.DB.UsePreparedStatements = true + } + + if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { + cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout + } + + if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { + cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout + } + + if cfg.Clients.GRPCKeepaliveParams != nil { + if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { + cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime + } + if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { + cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout + } + } + + if cfg.Server.Cache.Inmemory.NumCounters <= 0 { + cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters + } + if cfg.Server.Cache.Inmemory.MaxCost <= 0 { + cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost + } + if cfg.Server.Cache.Inmemory.BufferItems <= 0 { + cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems + } + + if len(cfg.Clients.SeqDB) == 0 { + defaultClient := SeqDBClient{ + ID: DefaultSeqDBClientID, + Timeout: cfg.Clients.SeqDBTimeout, + AvgDocSize: cfg.Clients.SeqDBAvgDocSize, + Addrs: cfg.Clients.SeqDBAddrs, + RequestRetries: cfg.Clients.RequestRetries, + InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, + MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, + ClientMode: cfg.Clients.ProxyClientMode, + GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, + } + cfg.Clients.SeqDB = []SeqDBClient{defaultClient} + } + + clientIDs := make(map[string]struct{}) + for _, client := range cfg.Clients.SeqDB { + if client.ID == "" { + return Config{}, fmt.Errorf("seq_db client ID cannot be empty") + } + if _, ok := clientIDs[client.ID]; ok { + return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) + } + clientIDs[client.ID] = struct{}{} + } + + if len(cfg.Handlers.SeqAPI.Envs) > 0 { + if cfg.Handlers.SeqAPI.DefaultEnv == "" { + return Config{}, fmt.Errorf("default_env must be specified when using envs") + } + + if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { + return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) + } + + for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { + if _, ok := clientIDs[envConfig.SeqDB]; !ok { + return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) + } + + if envConfig.Options == nil { + envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions + } else { + setSeqAPIOptionsDefaults(envConfig.Options) + } + + cfg.Handlers.SeqAPI.Envs[envName] = envConfig + } + } + + return cfg, nil +} + +func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { + if options.MaxAggregationsPerRequest <= 0 { + options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest + } + if options.MaxBucketsPerAggregationTs <= 0 { + options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs + } + if options.MaxParallelExportRequests <= 0 { + options.MaxParallelExportRequests = defaultMaxParallelExportRequests + } + if options.MaxSearchTotalLimit <= 0 { + options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit + } + if options.MaxSearchOffsetLimit <= 0 { + options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit + } + if options.MaxExportLimit <= 0 { + options.MaxExportLimit = defaultMaxExportLimit + } + if options.EventsCacheTTL <= 0 { + options.EventsCacheTTL = defaultEventsCacheTTL + } + if options.LogsLifespanCacheKey == "" { + options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey + } + if options.LogsLifespanCacheTTL <= 0 { + options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL + } +} + +func parse(cfg []byte) (Config, error) { + result := Config{} + + decoder := yaml.NewDecoder(bytes.NewReader(cfg)) + decoder.KnownFields(true) + if err := decoder.Decode(&result); err != nil { + return result, fmt.Errorf("error parsing config: %w", err) + } + + return result, nil +} diff --git a/internal/app/config/config_v2.go b/internal/app/config/config_v2.go new file mode 100644 index 0000000..f350a4e --- /dev/null +++ b/internal/app/config/config_v2.go @@ -0,0 +1,491 @@ +package config + +import ( + "bytes" + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + DefaultSeqDBClientID = "default" + + ProxyClientModeGRPC = "grpc" + + MaskModeMask = "mask" + MaskModeReplace = "replace" + MaskModeCut = "cut" + + FieldFilterConditionAnd = "and" + FieldFilterConditionOr = "or" + FieldFilterConditionNot = "not" + + FieldFilterModeEqual = "equal" + FieldFilterModeContains = "contains" + FieldFilterModePrefix = "prefix" + FieldFilterModeSuffix = "suffix" + + minGRPCKeepaliveTime = 10 * time.Second + minGRPCKeepaliveTimeout = 1 * time.Second + + defaultAsyncSearchListQueryLengthLimit = 1000 + + defaultMaxSearchTotalLimit = 1000000 + defaultMaxSearchOffsetLimit = 1000000 + defaultMaxExportLimit = 100000 + defaultMaxAggregationsPerRequest = 1 + defaultMaxBucketsPerAggregationTs = 200 + defaultMaxParallelExportRequests = 1 + + defaultInmemCacheNumCounters = 10000000 + defaultInmemCacheMaxCost = 1000000 + defaultInmemCacheBufferItems = 64 + + defaultEventsCacheTTL = 24 * time.Hour + + defaultLogsLifespanCacheKey = "logs_lifespan" + defaultLogsLifespanCacheTTL = 10 * time.Minute + + defaultClickHouseDialTimeout = 5 * time.Second + defaultClickHouseReadTimeout = 30 * time.Second +) + +type Config struct { + Server *Server `yaml:"server"` + Clients *Clients `yaml:"clients"` + Handlers *Handlers `yaml:"handlers"` + DB *DB `yaml:"db"` +} + +type CORS struct { + AllowedOrigins []string `yaml:"allowed_origins"` + AllowedMethods []string `yaml:"allowed_methods"` + AllowedHeaders []string `yaml:"allowed_headers"` + ExposedHeaders []string `yaml:"exposed_headers"` + AllowCredentials bool `yaml:"allow_credentials"` + MaxAge int `yaml:"max_age"` + OptionsPassthrough bool `yaml:"options_passthrough"` +} + +type OIDC struct { + SkipVerify bool `yaml:"skip_verify"` + AuthURLs []string `yaml:"auth_urls"` + RootCA string `yaml:"root_ca"` + CACert string `yaml:"ca_cert"` + PrivateKey string `yaml:"private_key"` + SSLSkipVerify bool `yaml:"ssl_skip_verify"` + AllowedClients []string `yaml:"allowed_clients"` + CacheSecretKey string `yaml:"cache_secret_key"` +} + +type DB struct { + Name string `yaml:"name"` + Host string `yaml:"host"` + Port int64 `yaml:"port"` + Pass string `yaml:"pass"` + User string `yaml:"user"` + RequestTimeout time.Duration `yaml:"request_timeout"` + ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` + UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` +} + +func (db *DB) ConnString() string { + return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) +} + +type CH struct { + Addrs []string `yaml:"addrs"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Sharded bool `yaml:"sharded"` + DialTimeout time.Duration `yaml:"dial_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` +} + +type ( + RateLimiter struct { + RatePerSec int `yaml:"rate_per_sec"` + MaxBurst int `yaml:"max_burst"` + StoreMaxKeys int `yaml:"store_max_keys"` + PerHandler bool `yaml:"per_handler"` + } + + UserToRateLimiter map[string]RateLimiter + + ApiRateLimiters struct { + Default RateLimiter `yaml:"default"` + SpecialUsers UserToRateLimiter `yaml:"spec_users"` + } + + ApiToRateLimiters map[string]ApiRateLimiters +) + +type InmemoryCache struct { + NumCounters int64 `yaml:"num_counters"` + MaxCost int64 `yaml:"max_cost"` + BufferItems int64 `yaml:"buffer_items"` +} + +type Redis struct { + Addr string `yaml:"addr"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Timeout time.Duration `yaml:"timeout"` + MaxRetries int `yaml:"max_retries"` + MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` +} + +type Cache struct { + Inmemory InmemoryCache `yaml:"inmemory"` + Redis *Redis `yaml:"redis"` +} + +type S3 struct { + Endpoint string `yaml:"endpoint"` + AccessKeyID string `yaml:"access_key_id"` + SecretAccessKey string `yaml:"secret_access_key"` + BucketName string `yaml:"bucket_name"` + EnableSSl bool `yaml:"enable_ssl"` +} + +type SeqProxyDownloader struct { + Delay time.Duration `yaml:"delay"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` +} + +type SessionStore struct { + Redis Redis `yaml:"redis"` + ExportLifetime time.Duration `yaml:"export_lifetime"` +} + +type FileStore struct { + S3 *S3 `yaml:"s3"` +} + +type MassExport struct { + BatchSize uint64 `yaml:"batch_size"` + WorkersCount int `yaml:"workers_count"` + TasksChannelSize int `yaml:"tasks_channel_size"` + PartLength time.Duration `yaml:"part_length"` + URLPrefix string `yaml:"url_prefix"` + AllowedUsers []string `yaml:"allowed_users"` + FileStore *FileStore `yaml:"file_store"` + SessionStore *SessionStore `yaml:"session_store"` + SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` +} + +type HTTP struct { + Addr string `yaml:"addr"` + ReadHeaderTimeout time.Duration `yaml:"read_header_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` + WriteTimeout time.Duration `yaml:"write_timeout"` + CORS *CORS `yaml:"cors"` +} + +type GRPC struct { + Addr string `yaml:"addr"` + ConnectionTimeout time.Duration `yaml:"connection_timeout"` +} + +type Debug struct { + Addr string `yaml:"addr"` +} + +type Auth struct { + OIDC *OIDC `yaml:"oidc"` + JWTSecretKey string `yaml:"jwt_secret_key"` +} + +type Server struct { + HTTP HTTP `yaml:"http"` + GRPC GRPC `yaml:"grpc"` + Debug Debug `yaml:"debug"` + Auth Auth `yaml:"auth"` + RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` + // CH *CH `yaml:"clickhouse"` + // Cache Cache `yaml:"cache"` +} + +type GRPCKeepaliveParams struct { + // After a duration of this time if the client doesn't see any activity it + // pings the server to see if the transport is still alive. + // If set below 10s, a minimum value of 10s will be used instead. + Time time.Duration `yaml:"time"` + // After having pinged for keepalive check, the client waits for a duration + // of Timeout and if no activity is seen even after that the connection is + // closed. If set below 1s, a minimum value of 1s will be used instead. + Timeout time.Duration `yaml:"timeout"` + // If true, client sends keepalive pings even with no active RPCs. If false, + // when there are no active RPCs, Time and Timeout will be ignored and no + // keepalive pings will be sent. False by default. + PermitWithoutStream bool `yaml:"permit_without_stream"` +} + +type SeqDBClient struct { + ID string `yaml:"id"` + Timeout time.Duration `yaml:"timeout"` + AvgDocSize int `yaml:"avg_doc_size"` + Addrs []string `yaml:"addrs"` + RequestRetries int `yaml:"request_retries"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + ClientMode string `yaml:"client_mode"` + GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` +} + +type Clients struct { + SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` + SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` + SeqDBAddrs []string `yaml:"seq_db_addrs"` + RequestRetries int `yaml:"request_retries"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + ProxyClientMode string `yaml:"proxy_client_mode"` + GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` + SeqDB []SeqDBClient `yaml:"seq_db"` +} + +type Handlers struct { + SeqAPI SeqAPI `yaml:"seq_api"` + ErrorGroups ErrorGroups `yaml:"error_groups"` + MassExport *MassExport `yaml:"mass_export"` + AsyncSearch AsyncSearch `yaml:"async_search"` +} + +type Field struct { + Name string `yaml:"name"` + Type string `yaml:"type"` +} + +type SeqAPI struct { + *SeqAPIOptions `yaml:",inline"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type SeqAPIEnv struct { + SeqDB string `yaml:"seq_db_id"` + Options *SeqAPIOptions `yaml:"options"` +} + +type SeqAPIOptions struct { + MaxSearchLimit int32 `yaml:"max_search_limit"` + MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` + MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` + MaxExportLimit int32 `yaml:"max_export_limit"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` + EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` + PinnedFields []Field `yaml:"pinned_fields"` + SystemFields []Field `yaml:"system_fields"` + LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` + LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` + FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` + Masking *Masking `yaml:"masking"` +} + +type Masking struct { + Masks []Mask `yaml:"masks"` + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` +} + +type Mask struct { + Re string `yaml:"re"` + Groups []int `yaml:"groups"` + Mode string `yaml:"mode"` // "mask" or "replace" or "cut" + ReplaceWord string `yaml:"replace_word"` // for mode:replace + + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` + + FieldFilters *FieldFilterSet `yaml:"field_filters"` +} + +type FieldFilter struct { + Field string `yaml:"field"` + Mode string `yaml:"mode"` // "equal" or "contains" or "prefix" or "suffix" + Values []string `yaml:"values"` +} + +type FieldFilterSet struct { + Condition string `yaml:"condition"` // "and" or "or" or "not" + Filters []FieldFilter `yaml:"filters"` // max 1 if condition:not +} + +type LogTagsMapping struct { + Release []string `yaml:"release"` + Env []string `yaml:"env"` +} + +type ErrorGroups struct { + LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` + QueryFilter map[string]string `yaml:"query_filter"` +} + +type AsyncSearch struct { + AdminUsers []string `yaml:"admin_users"` + ListQueryLengthLimit int `yaml:"list_query_length_limit"` +} + +// FromFile parse config from config path. +func FromFile(cfgPath string) (Config, error) { + cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec + if err != nil { + return Config{}, fmt.Errorf("error reading file: %s", err) + } + + cfg, err := parse(cfgBytes) + if err != nil { + return Config{}, fmt.Errorf("error parsing file: %s", err) + } + + proxyClientMode := cfg.Clients.ProxyClientMode + if proxyClientMode == "" { + cfg.Clients.ProxyClientMode = ProxyClientModeGRPC + } else if proxyClientMode != ProxyClientModeGRPC { + return Config{}, fmt.Errorf( + "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", + proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, + ) + } + + setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + + if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { + cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + } + + if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { + cfg.DB.UsePreparedStatements = new(bool) + *cfg.DB.UsePreparedStatements = true + } + + if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { + cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout + } + + if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { + cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout + } + + if cfg.Clients.GRPCKeepaliveParams != nil { + if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { + cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime + } + if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { + cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout + } + } + + if cfg.Server.Cache.Inmemory.NumCounters <= 0 { + cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters + } + if cfg.Server.Cache.Inmemory.MaxCost <= 0 { + cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost + } + if cfg.Server.Cache.Inmemory.BufferItems <= 0 { + cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems + } + + if len(cfg.Clients.SeqDB) == 0 { + defaultClient := SeqDBClient{ + ID: DefaultSeqDBClientID, + Timeout: cfg.Clients.SeqDBTimeout, + AvgDocSize: cfg.Clients.SeqDBAvgDocSize, + Addrs: cfg.Clients.SeqDBAddrs, + RequestRetries: cfg.Clients.RequestRetries, + InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, + MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, + ClientMode: cfg.Clients.ProxyClientMode, + GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, + } + cfg.Clients.SeqDB = []SeqDBClient{defaultClient} + } + + clientIDs := make(map[string]struct{}) + for _, client := range cfg.Clients.SeqDB { + if client.ID == "" { + return Config{}, fmt.Errorf("seq_db client ID cannot be empty") + } + if _, ok := clientIDs[client.ID]; ok { + return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) + } + clientIDs[client.ID] = struct{}{} + } + + if len(cfg.Handlers.SeqAPI.Envs) > 0 { + if cfg.Handlers.SeqAPI.DefaultEnv == "" { + return Config{}, fmt.Errorf("default_env must be specified when using envs") + } + + if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { + return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) + } + + for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { + if _, ok := clientIDs[envConfig.SeqDB]; !ok { + return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) + } + + if envConfig.Options == nil { + envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions + } else { + setSeqAPIOptionsDefaults(envConfig.Options) + } + + cfg.Handlers.SeqAPI.Envs[envName] = envConfig + } + } + + return cfg, nil +} + +func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { + if options.MaxAggregationsPerRequest <= 0 { + options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest + } + if options.MaxBucketsPerAggregationTs <= 0 { + options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs + } + if options.MaxParallelExportRequests <= 0 { + options.MaxParallelExportRequests = defaultMaxParallelExportRequests + } + if options.MaxSearchTotalLimit <= 0 { + options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit + } + if options.MaxSearchOffsetLimit <= 0 { + options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit + } + if options.MaxExportLimit <= 0 { + options.MaxExportLimit = defaultMaxExportLimit + } + if options.EventsCacheTTL <= 0 { + options.EventsCacheTTL = defaultEventsCacheTTL + } + if options.LogsLifespanCacheKey == "" { + options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey + } + if options.LogsLifespanCacheTTL <= 0 { + options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL + } +} + +func parse(cfg []byte) (Config, error) { + result := Config{} + + decoder := yaml.NewDecoder(bytes.NewReader(cfg)) + decoder.KnownFields(true) + if err := decoder.Decode(&result); err != nil { + return result, fmt.Errorf("error parsing config: %w", err) + } + + return result, nil +} diff --git a/internal/app/config/migrate.go b/internal/app/config/migrate.go new file mode 100644 index 0000000..d912156 --- /dev/null +++ b/internal/app/config/migrate.go @@ -0,0 +1 @@ +package config From 61143435b375e8b043f72936040b2d3539ae83da Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Mon, 13 Jul 2026 14:22:44 +0300 Subject: [PATCH 02/29] draft config v2 --- cmd/config-migrate/main.go | 48 ++ cmd/seq-ui/main.go | 8 +- docker-compose.yml | 2 +- internal/app/config/config.go | 359 ++------------ internal/app/config/migrate.go | 1 - internal/app/config/migrate_v1_v2.go | 443 ++++++++++++++++++ .../app/config/{config_v1.go => v1/config.go} | 214 +-------- .../app/config/{config_v2.go => v2/config.go} | 273 ++++++----- 8 files changed, 694 insertions(+), 654 deletions(-) create mode 100644 cmd/config-migrate/main.go delete mode 100644 internal/app/config/migrate.go create mode 100644 internal/app/config/migrate_v1_v2.go rename internal/app/config/{config_v1.go => v1/config.go} (60%) rename internal/app/config/{config_v2.go => v2/config.go} (70%) diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go new file mode 100644 index 0000000..08fd1b8 --- /dev/null +++ b/cmd/config-migrate/main.go @@ -0,0 +1,48 @@ +package main + +import ( + "errors" + "flag" + "os" + + "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/logger" + "go.uber.org/zap" +) + +var ( + source = flag.String("source", "", "path to the config file to migrate in place") +) + +func main() { + flag.Parse() + + run(*source) +} + +func run(source string) { + if source == "" { + logger.Fatal("missing required parameter", zap.String("param", "-source")) + } + + legacyCfg, err := os.ReadFile(source) + if err != nil { + logger.Fatal("error reading legacy config", zap.String("source", source), zap.Error(err)) + } + + backupPath := source + ".bak" + + if _, err := os.Stat(backupPath); err == nil { + logger.Fatal("backup already exists, remove it before re-running", zap.String("backup", backupPath)) + } else if !errors.Is(err, os.ErrNotExist) { + logger.Fatal("stat backup", zap.String("backup", backupPath), zap.Error(err)) + } + if err := os.WriteFile(backupPath, legacyCfg, 0644); err != nil { + logger.Fatal("error writing backup legacy config", zap.String("backup", backupPath), zap.Error(err)) + } + + backupCfg, err := config.FromFile(backupPath) + if err != nil { + logger.Fatal("read backup config file error", zap.Error(err)) + } +} diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 9450352..f587b15 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -24,7 +24,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/profiles" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/server" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" @@ -103,7 +103,7 @@ func run(ctx context.Context) { } } -func initApp(ctx context.Context, cfg config.Config) *api.Registrar { +func initApp(ctx context.Context, cfg config_v2.Config) *api.Registrar { logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { @@ -145,7 +145,7 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { } logger.Info("initializing db") - db, err := initDb(ctx, cfg.Server.DB) + db, err := initDb(ctx, cfg.DB) if err != nil { logger.Fatal("failed to init db", zap.Error(err)) } @@ -157,7 +157,7 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { dashboardsV1 *dashboards_v1.Dashboards ) if db != nil { - repo := repository.New(db, cfg.Server.DB.RequestTimeout) + repo := repository.New(db, cfg.DB.RequestTimeout) svc := service.New(repo) p = profiles.New(svc) diff --git a/docker-compose.yml b/docker-compose.yml index 37cf965..ee1ca48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.8' +version: "3.8" services: postgres: diff --git a/internal/app/config/config.go b/internal/app/config/config.go index a3b1a5b..b861a80 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -1,335 +1,72 @@ package config import ( + "bytes" "fmt" "os" - "time" + + v1 "github.com/ozontech/seq-ui/internal/app/config/v1" + v2 "github.com/ozontech/seq-ui/internal/app/config/v2" + "gopkg.in/yaml.v3" ) const ( - DefaultSeqDBClientID = "default" - - ProxyClientModeGRPC = "grpc" - - MaskModeMask = "mask" - MaskModeReplace = "replace" - MaskModeCut = "cut" - - FieldFilterConditionAnd = "and" - FieldFilterConditionOr = "or" - FieldFilterConditionNot = "not" - - FieldFilterModeEqual = "equal" - FieldFilterModeContains = "contains" - FieldFilterModePrefix = "prefix" - FieldFilterModeSuffix = "suffix" - - minGRPCKeepaliveTime = 10 * time.Second - minGRPCKeepaliveTimeout = 1 * time.Second - - defaultAsyncSearchListQueryLengthLimit = 1000 - - defaultMaxSearchTotalLimit = 1000000 - defaultMaxSearchOffsetLimit = 1000000 - defaultMaxExportLimit = 100000 - defaultMaxAggregationsPerRequest = 1 - defaultMaxBucketsPerAggregationTs = 200 - defaultMaxParallelExportRequests = 1 - - defaultInmemCacheNumCounters = 10000000 - defaultInmemCacheMaxCost = 1000000 - defaultInmemCacheBufferItems = 64 - - defaultEventsCacheTTL = 24 * time.Hour - - defaultLogsLifespanCacheKey = "logs_lifespan" - defaultLogsLifespanCacheTTL = 10 * time.Minute - - defaultClickHouseDialTimeout = 5 * time.Second - defaultClickHouseReadTimeout = 30 * time.Second + configV1Name = "config_v1" + configV2Name = "config_v2" + previousVersion = 1 + currentVersion = 2 ) -<<<<<<< HEAD -======= -type Config struct { - Server *Server `yaml:"server"` - Clients *Clients `yaml:"clients"` - Handlers *Handlers `yaml:"handlers"` -} - -type CORS struct { - AllowedOrigins []string `yaml:"allowed_origins"` - AllowedMethods []string `yaml:"allowed_methods"` - AllowedHeaders []string `yaml:"allowed_headers"` - ExposedHeaders []string `yaml:"exposed_headers"` - AllowCredentials bool `yaml:"allow_credentials"` - MaxAge int `yaml:"max_age"` - OptionsPassthrough bool `yaml:"options_passthrough"` -} - -type OIDC struct { - SkipVerify bool `yaml:"skip_verify"` - AuthURLs []string `yaml:"auth_urls"` - RootCA string `yaml:"root_ca"` - CACert string `yaml:"ca_cert"` - PrivateKey string `yaml:"private_key"` - SSLSkipVerify bool `yaml:"ssl_skip_verify"` - AllowedClients []string `yaml:"allowed_clients"` - CacheSecretKey string `yaml:"cache_secret_key"` +type configMeta struct { + Version *int `yaml:"version"` } -type DB struct { - Name string `yaml:"name"` - Host string `yaml:"host"` - Port int64 `yaml:"port"` - Pass string `yaml:"pass"` - User string `yaml:"user"` - RequestTimeout time.Duration `yaml:"request_timeout"` - ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` - UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` -} - -func (db *DB) ConnString() string { - return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) -} - -type CH struct { - Addrs []string `yaml:"addrs"` - Database string `yaml:"database"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Sharded bool `yaml:"sharded"` - DialTimeout time.Duration `yaml:"dial_timeout"` - ReadTimeout time.Duration `yaml:"read_timeout"` -} - -type ( - RateLimiter struct { - RatePerSec int `yaml:"rate_per_sec"` - MaxBurst int `yaml:"max_burst"` - StoreMaxKeys int `yaml:"store_max_keys"` - PerHandler bool `yaml:"per_handler"` +// FromFile parse config from config path. +func FromFile(cfgPath string) (v2.Config, error) { + cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec + if err != nil { + return v2.Config{}, fmt.Errorf("error reading file: %s", err) } - UserToRateLimiter map[string]RateLimiter - - ApiRateLimiters struct { - Default RateLimiter `yaml:"default"` - SpecialUsers UserToRateLimiter `yaml:"spec_users"` + meta, err := parse[configMeta](cfgBytes, false) + if err != nil { + return v2.Config{}, err } - ApiToRateLimiters map[string]ApiRateLimiters -) - -type InmemoryCache struct { - NumCounters int64 `yaml:"num_counters"` - MaxCost int64 `yaml:"max_cost"` - BufferItems int64 `yaml:"buffer_items"` -} - -type Redis struct { - Addr string `yaml:"addr"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Timeout time.Duration `yaml:"timeout"` - MaxRetries int `yaml:"max_retries"` - MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` -} - -type Cache struct { - Inmemory InmemoryCache `yaml:"inmemory"` - Redis *Redis `yaml:"redis"` -} - -type S3 struct { - Endpoint string `yaml:"endpoint"` - AccessKeyID string `yaml:"access_key_id"` - SecretAccessKey string `yaml:"secret_access_key"` - BucketName string `yaml:"bucket_name"` - EnableSSl bool `yaml:"enable_ssl"` -} - -type SeqProxyDownloader struct { - Delay time.Duration `yaml:"delay"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` -} - -type SessionStore struct { - Redis Redis `yaml:"redis"` - ExportLifetime time.Duration `yaml:"export_lifetime"` -} - -type FileStore struct { - S3 *S3 `yaml:"s3"` -} - -type MassExport struct { - BatchSize uint64 `yaml:"batch_size"` - WorkersCount int `yaml:"workers_count"` - TasksChannelSize int `yaml:"tasks_channel_size"` - PartLength time.Duration `yaml:"part_length"` - URLPrefix string `yaml:"url_prefix"` - AllowedUsers []string `yaml:"allowed_users"` - FileStore *FileStore `yaml:"file_store"` - SessionStore *SessionStore `yaml:"session_store"` - SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` -} - -type Server struct { - DebugAddr string `yaml:"debug_addr"` - HTTPAddr string `yaml:"http_addr"` - GRPCAddr string `yaml:"grpc_addr"` - CORS *CORS `yaml:"cors"` - OIDC *OIDC `yaml:"oidc"` - GRPCConnectionTimeout time.Duration `yaml:"grpc_connection_timeout"` - HTTPReadHeaderTimeout time.Duration `yaml:"http_read_header_timeout"` - HTTPReadTimeout time.Duration `yaml:"http_read_timeout"` - HTTPWriteTimeout time.Duration `yaml:"http_write_timeout"` - DB *DB `yaml:"db"` - CH *CH `yaml:"clickhouse"` - RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` - Cache Cache `yaml:"cache"` - JWTSecretKey string `yaml:"jwt_secret_key"` -} - -type GRPCKeepaliveParams struct { - // After a duration of this time if the client doesn't see any activity it - // pings the server to see if the transport is still alive. - // If set below 10s, a minimum value of 10s will be used instead. - Time time.Duration `yaml:"time"` - // After having pinged for keepalive check, the client waits for a duration - // of Timeout and if no activity is seen even after that the connection is - // closed. If set below 1s, a minimum value of 1s will be used instead. - Timeout time.Duration `yaml:"timeout"` - // If true, client sends keepalive pings even with no active RPCs. If false, - // when there are no active RPCs, Time and Timeout will be ignored and no - // keepalive pings will be sent. False by default. - PermitWithoutStream bool `yaml:"permit_without_stream"` -} - -type SeqDBClient struct { - ID string `yaml:"id"` - Timeout time.Duration `yaml:"timeout"` - AvgDocSize int `yaml:"avg_doc_size"` - Addrs []string `yaml:"addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ClientMode string `yaml:"client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` -} - -type Clients struct { - SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` - SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` - SeqDBAddrs []string `yaml:"seq_db_addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ProxyClientMode string `yaml:"proxy_client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` - SeqDB []SeqDBClient `yaml:"seq_db"` -} - -type Handlers struct { - SeqAPI SeqAPI `yaml:"seq_api"` - ErrorGroups ErrorGroups `yaml:"error_groups"` - MassExport *MassExport `yaml:"mass_export"` - AsyncSearch AsyncSearch `yaml:"async_search"` -} - -type Field struct { - Name string `yaml:"name"` - Type string `yaml:"type"` -} - -type SeqAPI struct { - *SeqAPIOptions `yaml:",inline"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` -} - -type SeqAPIEnv struct { - SeqDB string `yaml:"seq_db_id"` - Options *SeqAPIOptions `yaml:"options"` -} - -type SeqAPIOptions struct { - MaxSearchLimit int32 `yaml:"max_search_limit"` - MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` - MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` - MaxExportLimit int32 `yaml:"max_export_limit"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` - EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` - PinnedFields []Field `yaml:"pinned_fields"` - SystemFields []Field `yaml:"system_fields"` - LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` - LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` - FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` - Masking *Masking `yaml:"masking"` -} - -type Masking struct { - Masks []Mask `yaml:"masks"` - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` -} - -type Mask struct { - Re string `yaml:"re"` - Groups []int `yaml:"groups"` - Mode string `yaml:"mode"` // "mask" or "replace" or "cut" - ReplaceWord string `yaml:"replace_word"` // for mode:replace - - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` - - FieldFilters *FieldFilterSet `yaml:"field_filters"` -} - -type FieldFilter struct { - Field string `yaml:"field"` - Mode string `yaml:"mode"` // "equal" or "contains" or "prefix" or "suffix" - Values []string `yaml:"values"` -} - -type FieldFilterSet struct { - Condition string `yaml:"condition"` // "and" or "or" or "not" - Filters []FieldFilter `yaml:"filters"` // max 1 if condition:not -} + cfg := v2.Config{} + switch { + case meta.Version == nil || *meta.Version == previousVersion: + cfgV1, err := parse[v1.Config](cfgBytes, true) + if err != nil { + return v2.Config{}, err + } + cfg = migrateV1ToV2(cfgV1) + case *meta.Version == currentVersion: + cfg, err = parse[v2.Config](cfgBytes, true) + if err != nil { + return v2.Config{}, err + } + default: + return v2.Config{}, fmt.Errorf("unsupported config version: %d", *meta.Version) + } -type LogTagsMapping struct { - Env []string `yaml:"env"` - Service []string `yaml:"service"` - Release []string `yaml:"release"` -} + if err := v2.Normalize(&cfg); err != nil { + return v2.Config{}, fmt.Errorf("normalize config: %w") + } -type ErrorGroups struct { - LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` - QueryFilter map[string]string `yaml:"query_filter"` + return cfg, nil } -type AsyncSearch struct { - AdminUsers []string `yaml:"admin_users"` - ListQueryLengthLimit int `yaml:"list_query_length_limit"` -} +func parse[T any](cfg []byte, strict bool) (T, error) { + var result T -// FromFile parse config from config path. ->>>>>>> main -func FromFile(cfgPath string) (Config, error) { - cfgBytes, err := os.ReadFile(cfgPath) - if err != nil { - return Config{}, fmt.Errorf("error reading file: %s", err) + decoder := yaml.NewDecoder(bytes.NewReader(cfg)) + if strict { + decoder.KnownFields(true) + } + if err := decoder.Decode(&result); err != nil { + return result, fmt.Errorf("error parsing config: %w", err) } - return fromBytes(cfgBytes) -} - -func fromBytes(cfgBytes []byte) (Config, error) { - + return result, nil } diff --git a/internal/app/config/migrate.go b/internal/app/config/migrate.go deleted file mode 100644 index d912156..0000000 --- a/internal/app/config/migrate.go +++ /dev/null @@ -1 +0,0 @@ -package config diff --git a/internal/app/config/migrate_v1_v2.go b/internal/app/config/migrate_v1_v2.go new file mode 100644 index 0000000..8dc9df8 --- /dev/null +++ b/internal/app/config/migrate_v1_v2.go @@ -0,0 +1,443 @@ +package config + +import ( + v1 "github.com/ozontech/seq-ui/internal/app/config/v1" + v2 "github.com/ozontech/seq-ui/internal/app/config/v2" +) + +func migrateV1ToV2(src v1.Config) v2.Config { + dst := v2.Config{Version: currentVersion} + + dst.Server = migrateServer(src.Server) + dst.Cache = migrateCache(src.Server) + dst.Clients = migrateClients(src) + dst.DB = migrateDB(src.Server) + dst.Handlers = migrateHandlers(src.Handlers, &dst) + + return dst +} + +func migrateServer(src *v1.Server) *v2.Server { + dst := &v2.Server{ + HTTP: v2.HTTP{ + Addr: src.HTTPAddr, + ReadTimeout: src.HTTPReadTimeout, + ReadHeaderTimeout: src.HTTPReadHeaderTimeout, + WriteTimeout: src.HTTPWriteTimeout, + CORS: migrateCORS(src.CORS), + }, + GRPC: v2.GRPC{ + Addr: src.GRPCAddr, + ConnectionTimeout: src.GRPCConnectionTimeout, + }, + Debug: v2.Debug{ + Addr: src.DebugAddr, + }, + RateLimiters: migrateApiRateLimiters(src.RateLimiters), + } + + if src.OIDC != nil { + dst.Auth = &v2.Auth{ + OIDC: &v2.OIDC{ + SkipVerify: src.OIDC.SkipVerify, + AuthURLs: src.OIDC.AuthURLs, + RootCA: src.OIDC.RootCA, + CACert: src.OIDC.CACert, + PrivateKey: src.OIDC.PrivateKey, + SSLSkipVerify: src.OIDC.SSLSkipVerify, + AllowedClients: src.OIDC.AllowedClients, + CacheSecretKey: src.OIDC.CacheSecretKey, + }, + } + } + + if src.JWTSecretKey != "" { + dst.Auth.JWT = &v2.JWT{SecretKey: src.JWTSecretKey} + } + + return dst +} + +func migrateCORS(src *v1.CORS) *v2.CORS { + if src == nil { + return nil + } + + return &v2.CORS{ + AllowedOrigins: src.AllowedOrigins, + AllowedMethods: src.AllowedMethods, + AllowedHeaders: src.AllowedHeaders, + ExposedHeaders: src.ExposedHeaders, + AllowCredentials: src.AllowCredentials, + MaxAge: src.MaxAge, + OptionsPassthrough: src.OptionsPassthrough, + } +} + +func migrateApiRateLimiters(src v1.ApiToRateLimiters) v2.ApiToRateLimiters { + if src == nil { + return nil + } + + dst := make(v2.ApiToRateLimiters, len(src)) + for api, rl := range src { + dst[api] = v2.ApiRateLimiters{ + Default: v2.RateLimiter{ + RatePerSec: rl.Default.RatePerSec, + MaxBurst: rl.Default.MaxBurst, + StoreMaxKeys: rl.Default.StoreMaxKeys, + PerHandler: rl.Default.PerHandler, + }, + SpecialUsers: migrateUserToRateLimiter(rl.SpecialUsers), + } + } + + return dst +} + +func migrateUserToRateLimiter(src v1.UserToRateLimiter) v2.UserToRateLimiter { + if src == nil { + return nil + } + + dst := make(v2.UserToRateLimiter, len(src)) + for k, rl := range src { + dst[k] = v2.RateLimiter{ + RatePerSec: rl.RatePerSec, + MaxBurst: rl.MaxBurst, + StoreMaxKeys: rl.StoreMaxKeys, + PerHandler: rl.PerHandler, + } + } + + return dst +} + +func migrateGRPCKeepaliveParams(src *v1.GRPCKeepaliveParams) *v2.GRPCKeepaliveParams { + if src == nil { + return nil + } + + return &v2.GRPCKeepaliveParams{ + Time: src.Time, + Timeout: src.Timeout, + PermitWithoutStream: src.PermitWithoutStream, + } +} + +func migrateClients(src v1.Config) *v2.Clients { + dst := &v2.Clients{} + + if src.Clients != nil { + if len(src.Clients.SeqDB) > 0 { + dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) + for _, s := range src.Clients.SeqDB { + dst.SeqDB = append(dst.SeqDB, migrateSeqDBClient(s)) + } + } else { + dst.SeqDB = []v2.SeqDBClient{{ + ID: v2.DefaultSeqDBClientID, + Timeout: src.Clients.SeqDBTimeout, + AvgDocSize: src.Clients.SeqDBAvgDocSize, + Addrs: src.Clients.SeqDBAddrs, + RequestRetries: src.Clients.RequestRetries, + InitialRetryBackoff: src.Clients.InitialRetryBackoff, + MaxRetryBackoff: src.Clients.MaxRetryBackoff, + ClientMode: src.Clients.ProxyClientMode, + GRPCKeepaliveParams: migrateGRPCKeepaliveParams(src.Clients.GRPCKeepaliveParams), + }} + } + } + + if src.Server != nil && src.Server.CH != nil { + ch := src.Server.CH + dst.ClickHouse = []v2.CHClient{{ + ID: v2.DefaultSeqDBClientID, + Addrs: ch.Addrs, + Database: ch.Database, + Username: ch.Username, + Password: ch.Password, + Sharded: ch.Sharded, + DialTimeout: ch.DialTimeout, + ReadTimeout: ch.ReadTimeout, + }} + } + + return dst +} + +func migrateSeqDBClient(src v1.SeqDBClient) v2.SeqDBClient { + return v2.SeqDBClient{ + ID: src.ID, + Timeout: src.Timeout, + AvgDocSize: src.AvgDocSize, + Addrs: src.Addrs, + RequestRetries: src.RequestRetries, + InitialRetryBackoff: src.InitialRetryBackoff, + MaxRetryBackoff: src.MaxRetryBackoff, + ClientMode: src.ClientMode, + GRPCKeepaliveParams: migrateGRPCKeepaliveParams(src.GRPCKeepaliveParams), + } +} + +func migrateDB(src *v1.Server) *v2.DB { + if src == nil || src.DB == nil { + return nil + } + + return &v2.DB{ + Name: src.DB.Name, + Host: src.DB.Host, + Port: src.DB.Port, + Pass: src.DB.Pass, + User: src.DB.User, + RequestTimeout: src.DB.RequestTimeout, + ConnectionPoolCapacity: src.DB.ConnectionPoolCapacity, + UsePreparedStatements: src.DB.UsePreparedStatements, + } +} + +func migrateCache(src *v1.Server) *v2.Cache { + if src == nil { + return &v2.Cache{} + } + + cache := &v2.Cache{} + cache.Inmemory = append(cache.Inmemory, v2.InmemoryCache{ + ID: v2.DefaultInmemCacheID, + NumCounters: src.Cache.Inmemory.NumCounters, + MaxCost: src.Cache.Inmemory.MaxCost, + BufferItems: src.Cache.Inmemory.BufferItems, + }) + + if src.Cache.Redis != nil { + cache.Redis = append(cache.Redis, migrateRedis(src.Cache.Redis, v2.DefaultRedisID, v2.DefaultInmemCacheID)) + } + + return cache +} + +func migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { + return v2.Redis{ + ID: id, + WithInmemID: withInmemID, + Addr: src.Addr, + Username: src.Username, + Password: src.Password, + Timeout: src.Timeout, + MaxRetries: src.MaxRetries, + MinRetryBackoff: src.MinRetryBackoff, + MaxRetryBackoff: src.MaxRetryBackoff, + } +} + +func migrateHandlers(src *v1.Handlers, cfg *v2.Config) *v2.Handlers { + if src == nil { + return &v2.Handlers{} + } + + dst := &v2.Handlers{ + SeqAPI: migrateSeqAPI(src.SeqAPI), + ErrorGroups: migrateErrorGroups(src.ErrorGroups), + AsyncSearch: migrateAsyncSearch(src.AsyncSearch), + } + + if src.MassExport != nil { + dst.MassExport = migrateMassExport(src.MassExport, cfg) + } + + return dst +} + +func migrateSeqAPI(src v1.SeqAPI) v2.SeqAPI { + return v2.SeqAPI{ + SeqAPIOptions: migrateSeqAPIOptions(src.SeqAPIOptions), + Envs: migrateSeqAPIEnvs(src.Envs), + DefaultEnv: src.DefaultEnv, + } +} + +func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv) map[string]v2.SeqAPIEnv { + if len(envs) == 0 { + return nil + } + + dst := make(map[string]v2.SeqAPIEnv, len(envs)) + for name, cfg := range envs { + dst[name] = v2.SeqAPIEnv{ + SeqDB: cfg.SeqDB, + Options: migrateSeqAPIOptions(cfg.Options), + } + } + + return dst +} + +func migrateSeqAPIOptions(options *v1.SeqAPIOptions) *v2.SeqAPIOptions { + if options == nil { + return nil + } + + return &v2.SeqAPIOptions{ + MaxSearchLimit: options.MaxSearchLimit, + MaxSearchTotalLimit: options.MaxSearchTotalLimit, + MaxSearchOffsetLimit: options.MaxSearchOffsetLimit, + MaxExportLimit: options.MaxExportLimit, + SeqCLIMaxSearchLimit: options.SeqCLIMaxSearchLimit, + MaxParallelExportRequests: options.MaxParallelExportRequests, + MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, + MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, + EventsCacheTTL: options.EventsCacheTTL, + PinnedFields: migrateFields(options.PinnedFields), + SystemFields: migrateFields(options.SystemFields), + LogsLifespanCacheKey: options.LogsLifespanCacheKey, + LogsLifespanCacheTTL: options.LogsLifespanCacheTTL, + FieldsCacheTTL: options.FieldsCacheTTL, + Masking: migrateMasking(options.Masking), + } +} + +func migrateFields(fs []v1.Field) []v2.Field { + if fs == nil { + return nil + } + + dst := make([]v2.Field, len(fs)) + for i, f := range fs { + dst[i] = v2.Field{ + Name: f.Name, + Type: f.Type, + } + } + + return dst +} + +func migrateMasking(src *v1.Masking) *v2.Masking { + if src == nil { + return nil + } + + return &v2.Masking{ + Masks: migrateMasks(src.Masks), + ProcessFields: src.ProcessFields, + IgnoreFields: src.IgnoreFields, + } +} + +func migrateMasks(ms []v1.Mask) []v2.Mask { + if ms == nil { + return nil + } + + dst := make([]v2.Mask, len(ms)) + for i, m := range ms { + dst[i] = v2.Mask{ + Re: m.Re, + Groups: m.Groups, + Mode: m.Mode, + ReplaceWord: m.ReplaceWord, + ProcessFields: m.ProcessFields, + IgnoreFields: m.IgnoreFields, + FieldFilters: migrateFieldFilters(m.FieldFilters), + } + } + + return dst +} + +func migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { + if src == nil { + return nil + } + + dst := &v2.FieldFilterSet{ + Condition: src.Condition, + } + + if src.Filters != nil { + dst.Filters = make([]v2.FieldFilter, len(src.Filters)) + for i, f := range src.Filters { + dst.Filters[i] = v2.FieldFilter{ + Field: f.Field, + Mode: f.Mode, + Values: f.Values, + } + } + } + + return dst +} + +func migrateErrorGroups(eg v1.ErrorGroups) v2.ErrorGroups { + return v2.ErrorGroups{ + LogTagsMapping: v2.LogTagsMapping{ + Env: eg.LogTagsMapping.Env, + Service: eg.LogTagsMapping.Service, + Release: eg.LogTagsMapping.Release, + }, + QueryFilter: eg.QueryFilter, + } +} + +func migrateAsyncSearch(a v1.AsyncSearch) v2.AsyncSearch { + return v2.AsyncSearch{ + AdminUsers: a.AdminUsers, + ListQueryLengthLimit: a.ListQueryLengthLimit, + } +} + +func migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { + dst := &v2.MassExport{ + BatchSize: me.BatchSize, + WorkersCount: me.WorkersCount, + TasksChannelSize: me.TasksChannelSize, + PartLength: me.PartLength, + URLPrefix: me.URLPrefix, + AllowedUsers: me.AllowedUsers, + FileStore: migrateFileStore(me.FileStore), + DownloadParams: migrateDownloadParams(me.SeqProxyDownloader), + } + + if me.SessionStore != nil { + cfg.Cache.Redis = append(cfg.Cache.Redis, migrateRedis(&me.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) + + dst.SessionStore = &v2.SessionStore{ + RedisID: v2.DefaultMassExportRedisID, + ExportLifetime: me.SessionStore.ExportLifetime, + } + } + + return dst +} + +func migrateFileStore(fs *v1.FileStore) *v2.FileStore { + if fs == nil { + return nil + } + + dst := &v2.FileStore{} + if fs.S3 != nil { + dst.S3 = &v2.S3{ + Endpoint: fs.S3.Endpoint, + AccessKeyID: fs.S3.AccessKeyID, + SecretAccessKey: fs.S3.SecretAccessKey, + BucketName: fs.S3.BucketName, + EnableSSl: fs.S3.EnableSSl, + } + } + + return dst +} + +func migrateDownloadParams(src *v1.SeqProxyDownloader) *v2.DownloadParams { + if src == nil { + return nil + } + + return &v2.DownloadParams{ + Delay: src.Delay, + InitialRetryBackoff: src.InitialRetryBackoff, + MaxRetryBackoff: src.MaxRetryBackoff, + } +} diff --git a/internal/app/config/config_v1.go b/internal/app/config/v1/config.go similarity index 60% rename from internal/app/config/config_v1.go rename to internal/app/config/v1/config.go index 589e661..9851298 100644 --- a/internal/app/config/config_v1.go +++ b/internal/app/config/v1/config.go @@ -1,59 +1,11 @@ -package config +package v1 import ( - "bytes" - "fmt" - "os" "time" - - "gopkg.in/yaml.v3" -) - -const ( - DefaultSeqDBClientID = "default" - - ProxyClientModeGRPC = "grpc" - - MaskModeMask = "mask" - MaskModeReplace = "replace" - MaskModeCut = "cut" - - FieldFilterConditionAnd = "and" - FieldFilterConditionOr = "or" - FieldFilterConditionNot = "not" - - FieldFilterModeEqual = "equal" - FieldFilterModeContains = "contains" - FieldFilterModePrefix = "prefix" - FieldFilterModeSuffix = "suffix" - - minGRPCKeepaliveTime = 10 * time.Second - minGRPCKeepaliveTimeout = 1 * time.Second - - defaultAsyncSearchListQueryLengthLimit = 1000 - - defaultMaxSearchTotalLimit = 1000000 - defaultMaxSearchOffsetLimit = 1000000 - defaultMaxExportLimit = 100000 - defaultMaxAggregationsPerRequest = 1 - defaultMaxBucketsPerAggregationTs = 200 - defaultMaxParallelExportRequests = 1 - - defaultInmemCacheNumCounters = 10000000 - defaultInmemCacheMaxCost = 1000000 - defaultInmemCacheBufferItems = 64 - - defaultEventsCacheTTL = 24 * time.Hour - - defaultLogsLifespanCacheKey = "logs_lifespan" - defaultLogsLifespanCacheTTL = 10 * time.Minute - - defaultClickHouseDialTimeout = 5 * time.Second - defaultClickHouseReadTimeout = 30 * time.Second ) type Config struct { - Version *int `yaml:"version,omitempty"` + Version *int `yaml:"version"` Server *Server `yaml:"server"` Clients *Clients `yaml:"clients"` Handlers *Handlers `yaml:"handlers"` @@ -91,10 +43,6 @@ type DB struct { UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` } -func (db *DB) ConnString() string { - return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) -} - type CH struct { Addrs []string `yaml:"addrs"` Database string `yaml:"database"` @@ -306,8 +254,9 @@ type FieldFilterSet struct { } type LogTagsMapping struct { - Release []string `yaml:"release"` Env []string `yaml:"env"` + Service []string `yaml:"service"` + Release []string `yaml:"release"` } type ErrorGroups struct { @@ -319,158 +268,3 @@ type AsyncSearch struct { AdminUsers []string `yaml:"admin_users"` ListQueryLengthLimit int `yaml:"list_query_length_limit"` } - -// FromFile parse config from config path. -func FromFile(cfgPath string) (Config, error) { - cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec - if err != nil { - return Config{}, fmt.Errorf("error reading file: %s", err) - } - - cfg, err := parse(cfgBytes) - if err != nil { - return Config{}, fmt.Errorf("error parsing file: %s", err) - } - - proxyClientMode := cfg.Clients.ProxyClientMode - if proxyClientMode == "" { - cfg.Clients.ProxyClientMode = ProxyClientModeGRPC - } else if proxyClientMode != ProxyClientModeGRPC { - return Config{}, fmt.Errorf( - "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", - proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, - ) - } - - setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) - - if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { - cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit - } - - if cfg.Server.DB != nil && cfg.Server.DB.UsePreparedStatements == nil { - cfg.Server.DB.UsePreparedStatements = new(bool) - *cfg.Server.DB.UsePreparedStatements = true - } - - if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { - cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout - } - - if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { - cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout - } - - if cfg.Clients.GRPCKeepaliveParams != nil { - if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { - cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime - } - if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { - cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout - } - } - - if cfg.Server.Cache.Inmemory.NumCounters <= 0 { - cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters - } - if cfg.Server.Cache.Inmemory.MaxCost <= 0 { - cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost - } - if cfg.Server.Cache.Inmemory.BufferItems <= 0 { - cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems - } - - if len(cfg.Clients.SeqDB) == 0 { - defaultClient := SeqDBClient{ - ID: DefaultSeqDBClientID, - Timeout: cfg.Clients.SeqDBTimeout, - AvgDocSize: cfg.Clients.SeqDBAvgDocSize, - Addrs: cfg.Clients.SeqDBAddrs, - RequestRetries: cfg.Clients.RequestRetries, - InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, - MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, - ClientMode: cfg.Clients.ProxyClientMode, - GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, - } - cfg.Clients.SeqDB = []SeqDBClient{defaultClient} - } - - clientIDs := make(map[string]struct{}) - for _, client := range cfg.Clients.SeqDB { - if client.ID == "" { - return Config{}, fmt.Errorf("seq_db client ID cannot be empty") - } - if _, ok := clientIDs[client.ID]; ok { - return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) - } - clientIDs[client.ID] = struct{}{} - } - - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return Config{}, fmt.Errorf("default_env must be specified when using envs") - } - - if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) - } - - for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if _, ok := clientIDs[envConfig.SeqDB]; !ok { - return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) - } - - if envConfig.Options == nil { - envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions - } else { - setSeqAPIOptionsDefaults(envConfig.Options) - } - - cfg.Handlers.SeqAPI.Envs[envName] = envConfig - } - } - - return cfg, nil -} - -func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { - if options.MaxAggregationsPerRequest <= 0 { - options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest - } - if options.MaxBucketsPerAggregationTs <= 0 { - options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs - } - if options.MaxParallelExportRequests <= 0 { - options.MaxParallelExportRequests = defaultMaxParallelExportRequests - } - if options.MaxSearchTotalLimit <= 0 { - options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit - } - if options.MaxSearchOffsetLimit <= 0 { - options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit - } - if options.MaxExportLimit <= 0 { - options.MaxExportLimit = defaultMaxExportLimit - } - if options.EventsCacheTTL <= 0 { - options.EventsCacheTTL = defaultEventsCacheTTL - } - if options.LogsLifespanCacheKey == "" { - options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey - } - if options.LogsLifespanCacheTTL <= 0 { - options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL - } -} - -func parse(cfg []byte) (Config, error) { - result := Config{} - - decoder := yaml.NewDecoder(bytes.NewReader(cfg)) - decoder.KnownFields(true) - if err := decoder.Decode(&result); err != nil { - return result, fmt.Errorf("error parsing config: %w", err) - } - - return result, nil -} diff --git a/internal/app/config/config_v2.go b/internal/app/config/v2/config.go similarity index 70% rename from internal/app/config/config_v2.go rename to internal/app/config/v2/config.go index f350a4e..882c00a 100644 --- a/internal/app/config/config_v2.go +++ b/internal/app/config/v2/config.go @@ -1,16 +1,15 @@ -package config +package v2 import ( - "bytes" "fmt" - "os" "time" - - "gopkg.in/yaml.v3" ) const ( - DefaultSeqDBClientID = "default" + DefaultSeqDBClientID = "default" + DefaultInmemCacheID = "seqapi" + DefaultRedisID = "default" + DefaultMassExportRedisID = "mass_export" ProxyClientModeGRPC = "grpc" @@ -53,10 +52,12 @@ const ( ) type Config struct { + Version int `yaml:"version"` Server *Server `yaml:"server"` Clients *Clients `yaml:"clients"` Handlers *Handlers `yaml:"handlers"` DB *DB `yaml:"db"` + Cache *Cache `yaml:"cache"` } type CORS struct { @@ -95,16 +96,6 @@ func (db *DB) ConnString() string { return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) } -type CH struct { - Addrs []string `yaml:"addrs"` - Database string `yaml:"database"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Sharded bool `yaml:"sharded"` - DialTimeout time.Duration `yaml:"dial_timeout"` - ReadTimeout time.Duration `yaml:"read_timeout"` -} - type ( RateLimiter struct { RatePerSec int `yaml:"rate_per_sec"` @@ -124,12 +115,15 @@ type ( ) type InmemoryCache struct { - NumCounters int64 `yaml:"num_counters"` - MaxCost int64 `yaml:"max_cost"` - BufferItems int64 `yaml:"buffer_items"` + ID string `yaml:"id"` + NumCounters int64 `yaml:"num_counters"` + MaxCost int64 `yaml:"max_cost"` + BufferItems int64 `yaml:"buffer_items"` } type Redis struct { + ID string `yaml:"id"` + WithInmemID string `yaml:"with_inmem_id"` Addr string `yaml:"addr"` Username string `yaml:"username"` Password string `yaml:"password"` @@ -140,8 +134,8 @@ type Redis struct { } type Cache struct { - Inmemory InmemoryCache `yaml:"inmemory"` - Redis *Redis `yaml:"redis"` + Inmemory []InmemoryCache `yaml:"inmemory"` + Redis []Redis `yaml:"redis"` } type S3 struct { @@ -152,14 +146,14 @@ type S3 struct { EnableSSl bool `yaml:"enable_ssl"` } -type SeqProxyDownloader struct { +type DownloadParams struct { Delay time.Duration `yaml:"delay"` InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` } type SessionStore struct { - Redis Redis `yaml:"redis"` + RedisID string `yaml:"redis_id"` ExportLifetime time.Duration `yaml:"export_lifetime"` } @@ -168,15 +162,15 @@ type FileStore struct { } type MassExport struct { - BatchSize uint64 `yaml:"batch_size"` - WorkersCount int `yaml:"workers_count"` - TasksChannelSize int `yaml:"tasks_channel_size"` - PartLength time.Duration `yaml:"part_length"` - URLPrefix string `yaml:"url_prefix"` - AllowedUsers []string `yaml:"allowed_users"` - FileStore *FileStore `yaml:"file_store"` - SessionStore *SessionStore `yaml:"session_store"` - SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` + BatchSize uint64 `yaml:"batch_size"` + WorkersCount int `yaml:"workers_count"` + TasksChannelSize int `yaml:"tasks_channel_size"` + PartLength time.Duration `yaml:"part_length"` + URLPrefix string `yaml:"url_prefix"` + AllowedUsers []string `yaml:"allowed_users"` + FileStore *FileStore `yaml:"file_store"` + SessionStore *SessionStore `yaml:"session_store"` + DownloadParams *DownloadParams `yaml:"download_params"` } type HTTP struct { @@ -196,19 +190,21 @@ type Debug struct { Addr string `yaml:"addr"` } +type JWT struct { + SecretKey string `yaml:"secret_key"` +} + type Auth struct { - OIDC *OIDC `yaml:"oidc"` - JWTSecretKey string `yaml:"jwt_secret_key"` + OIDC *OIDC `yaml:"oidc"` + JWT *JWT `yaml:"jwt"` } type Server struct { HTTP HTTP `yaml:"http"` GRPC GRPC `yaml:"grpc"` Debug Debug `yaml:"debug"` - Auth Auth `yaml:"auth"` + Auth *Auth `yaml:"auth"` RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` - // CH *CH `yaml:"clickhouse"` - // Cache Cache `yaml:"cache"` } type GRPCKeepaliveParams struct { @@ -236,18 +232,23 @@ type SeqDBClient struct { MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` ClientMode string `yaml:"client_mode"` GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` + DownloadParams *DownloadParams `yaml:"download_params"` +} + +type CHClient struct { + ID string `yaml:"id"` + Addrs []string `yaml:"addrs"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Sharded bool `yaml:"sharded"` + DialTimeout time.Duration `yaml:"dial_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` } type Clients struct { - SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` - SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` - SeqDBAddrs []string `yaml:"seq_db_addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ProxyClientMode string `yaml:"proxy_client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` - SeqDB []SeqDBClient `yaml:"seq_db"` + SeqDB []SeqDBClient `yaml:"seq_db"` + ClickHouse []CHClient `yaml:"clickhouse"` } type Handlers struct { @@ -322,6 +323,7 @@ type FieldFilterSet struct { type LogTagsMapping struct { Release []string `yaml:"release"` + Service []string `yaml:"service"` Env []string `yaml:"env"` } @@ -335,104 +337,124 @@ type AsyncSearch struct { ListQueryLengthLimit int `yaml:"list_query_length_limit"` } -// FromFile parse config from config path. -func FromFile(cfgPath string) (Config, error) { - cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec - if err != nil { - return Config{}, fmt.Errorf("error reading file: %s", err) +func Normalize(cfg *Config) error { + if len(cfg.Clients.SeqDB) == 0 { + return fmt.Errorf("clients.seq_db must contain at least one client") } - cfg, err := parse(cfgBytes) - if err != nil { - return Config{}, fmt.Errorf("error parsing file: %s", err) - } + seqDBIDs := make(map[string]struct{}, len(cfg.Clients.SeqDB)) + for i := range cfg.Clients.SeqDB { + c := &cfg.Clients.SeqDB[i] + if c.ID == "" { + return fmt.Errorf("seq_db client ID cannot be empty") + } + if _, ok := seqDBIDs[c.ID]; ok { + return fmt.Errorf("duplicate seq_db client ID: %s", c.ID) + } - proxyClientMode := cfg.Clients.ProxyClientMode - if proxyClientMode == "" { - cfg.Clients.ProxyClientMode = ProxyClientModeGRPC - } else if proxyClientMode != ProxyClientModeGRPC { - return Config{}, fmt.Errorf( - "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", - proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, - ) - } + seqDBIDs[c.ID] = struct{}{} - setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + if c.ClientMode == "" { + c.ClientMode = ProxyClientModeGRPC + } else if c.ClientMode != ProxyClientModeGRPC { + return fmt.Errorf("invalid clients.seq_db[%s].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, ProxyClientModeGRPC) + } - if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { - cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + if c.GRPCKeepaliveParams != nil { + if c.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { + c.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime + } + if c.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { + c.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout + } + } } - if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { - cfg.DB.UsePreparedStatements = new(bool) - *cfg.DB.UsePreparedStatements = true - } + chIDs := make(map[string]struct{}, len(cfg.Clients.ClickHouse)) + for i := range cfg.Clients.ClickHouse { + ch := &cfg.Clients.ClickHouse[i] + if ch.ID == "" { + return fmt.Errorf("clickhouse client ID cannot be empty") + } + if _, ok := chIDs[ch.ID]; ok { + return fmt.Errorf("duplicate clickhouse client ID: %s", ch.ID) + } - if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { - cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout - } + chIDs[ch.ID] = struct{}{} - if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { - cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout + if ch.DialTimeout <= 0 { + ch.DialTimeout = defaultClickHouseDialTimeout + } + if ch.ReadTimeout <= 0 { + ch.ReadTimeout = defaultClickHouseReadTimeout + } } - if cfg.Clients.GRPCKeepaliveParams != nil { - if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { - cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime + inmemIDs := make(map[string]struct{}, len(cfg.Cache.Inmemory)) + for i := range cfg.Cache.Inmemory { + inm := &cfg.Cache.Inmemory[i] + if inm.ID == "" { + return fmt.Errorf("inmemory cache ID cannot be empty") } - if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { - cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout + if _, ok := inmemIDs[inm.ID]; ok { + return fmt.Errorf("duplicate inmemory cache ID: %s", inm.ID) } - } - if cfg.Server.Cache.Inmemory.NumCounters <= 0 { - cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters - } - if cfg.Server.Cache.Inmemory.MaxCost <= 0 { - cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost - } - if cfg.Server.Cache.Inmemory.BufferItems <= 0 { - cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems - } + inmemIDs[inm.ID] = struct{}{} - if len(cfg.Clients.SeqDB) == 0 { - defaultClient := SeqDBClient{ - ID: DefaultSeqDBClientID, - Timeout: cfg.Clients.SeqDBTimeout, - AvgDocSize: cfg.Clients.SeqDBAvgDocSize, - Addrs: cfg.Clients.SeqDBAddrs, - RequestRetries: cfg.Clients.RequestRetries, - InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, - MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, - ClientMode: cfg.Clients.ProxyClientMode, - GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, + if inm.NumCounters <= 0 { + inm.NumCounters = defaultInmemCacheNumCounters + } + if inm.MaxCost <= 0 { + inm.MaxCost = defaultInmemCacheMaxCost + } + if inm.BufferItems <= 0 { + inm.BufferItems = defaultInmemCacheBufferItems } - cfg.Clients.SeqDB = []SeqDBClient{defaultClient} } - clientIDs := make(map[string]struct{}) - for _, client := range cfg.Clients.SeqDB { - if client.ID == "" { - return Config{}, fmt.Errorf("seq_db client ID cannot be empty") + redisIDs := make(map[string]struct{}, len(cfg.Cache.Redis)) + for i := range cfg.Cache.Redis { + r := &cfg.Cache.Redis[i] + if r.ID == "" { + return fmt.Errorf("redis cache ID cannot be empty") } - if _, ok := clientIDs[client.ID]; ok { - return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) + if _, ok := inmemIDs[r.ID]; ok { + return fmt.Errorf("duplicate redis cache ID: %s", r.ID) } - clientIDs[client.ID] = struct{}{} + + redisIDs[r.ID] = struct{}{} + + if r.WithInmemID != "" { + if _, ok := redisIDs[r.WithInmemID]; !ok { + return fmt.Errorf("redis cache %q references unknown inmem cache id %q", r.ID, r.WithInmemID) + } + } + } + + if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { + cfg.DB.UsePreparedStatements = new(bool) + *cfg.DB.UsePreparedStatements = true + } + + if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { + cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit } + setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + if len(cfg.Handlers.SeqAPI.Envs) > 0 { if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return Config{}, fmt.Errorf("default_env must be specified when using envs") + return fmt.Errorf("default_env must be specified when using envs") } if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) + return fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) } for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if _, ok := clientIDs[envConfig.SeqDB]; !ok { - return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) + if _, ok := seqDBIDs[envConfig.SeqDB]; !ok { + return fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) } if envConfig.Options == nil { @@ -445,7 +467,16 @@ func FromFile(cfgPath string) (Config, error) { } } - return cfg, nil + if cfg.Handlers.MassExport != nil { + if cfg.Handlers.MassExport.SessionStore.RedisID == "" { + return fmt.Errorf("handlers.mass_export.session_store.redis_id cannot be empty") + } + + if _, ok := redisIDs[cfg.Handlers.MassExport.SessionStore.RedisID]; !ok { + return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) + } + } + return nil } func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { @@ -477,15 +508,3 @@ func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL } } - -func parse(cfg []byte) (Config, error) { - result := Config{} - - decoder := yaml.NewDecoder(bytes.NewReader(cfg)) - decoder.KnownFields(true) - if err := decoder.Decode(&result); err != nil { - return result, fmt.Errorf("error parsing config: %w", err) - } - - return result, nil -} From 4dc75725b8693e504548d00fe2cfefca6fe23886 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 22 Jul 2026 12:44:28 +0300 Subject: [PATCH 03/29] fix --- cmd/seq-ui/m.md | 0 config/config.example.yaml | 2 - config/config.multicluster.yaml | 2 - internal/app/config/config.go | 3 +- .../{migrate_v1_v2.go => migrate/v1_v2.go} | 6 +- internal/app/config/v1/config.go | 210 +++++++++++++++++ internal/app/config/v2/config.go | 218 +++++++++++++++++- 7 files changed, 427 insertions(+), 14 deletions(-) create mode 100644 cmd/seq-ui/m.md rename internal/app/config/{migrate_v1_v2.go => migrate/v1_v2.go} (99%) diff --git a/cmd/seq-ui/m.md b/cmd/seq-ui/m.md new file mode 100644 index 0000000..e69de29 diff --git a/config/config.example.yaml b/config/config.example.yaml index 8e0a5fa..604c16c 100644 --- a/config/config.example.yaml +++ b/config/config.example.yaml @@ -1,5 +1,3 @@ -version: 2 - server: http_addr: ":5555" grpc_addr: ":5556" diff --git a/config/config.multicluster.yaml b/config/config.multicluster.yaml index 3d1eb81..94fb312 100644 --- a/config/config.multicluster.yaml +++ b/config/config.multicluster.yaml @@ -1,5 +1,3 @@ -version: 2 - server: http_addr: ":5555" grpc_addr: ":5556" diff --git a/internal/app/config/config.go b/internal/app/config/config.go index b861a80..9f46c43 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/ozontech/seq-ui/internal/app/config/migrate" v1 "github.com/ozontech/seq-ui/internal/app/config/v1" v2 "github.com/ozontech/seq-ui/internal/app/config/v2" "gopkg.in/yaml.v3" @@ -40,7 +41,7 @@ func FromFile(cfgPath string) (v2.Config, error) { if err != nil { return v2.Config{}, err } - cfg = migrateV1ToV2(cfgV1) + cfg = migrate.V1ToV2(cfgV1) case *meta.Version == currentVersion: cfg, err = parse[v2.Config](cfgBytes, true) if err != nil { diff --git a/internal/app/config/migrate_v1_v2.go b/internal/app/config/migrate/v1_v2.go similarity index 99% rename from internal/app/config/migrate_v1_v2.go rename to internal/app/config/migrate/v1_v2.go index 8dc9df8..522fc50 100644 --- a/internal/app/config/migrate_v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -1,12 +1,12 @@ -package config +package migrate import ( v1 "github.com/ozontech/seq-ui/internal/app/config/v1" v2 "github.com/ozontech/seq-ui/internal/app/config/v2" ) -func migrateV1ToV2(src v1.Config) v2.Config { - dst := v2.Config{Version: currentVersion} +func V1ToV2(src v1.Config) v2.Config { + dst := v2.Config{Version: 2} dst.Server = migrateServer(src.Server) dst.Cache = migrateCache(src.Server) diff --git a/internal/app/config/v1/config.go b/internal/app/config/v1/config.go index 9851298..e3deb74 100644 --- a/internal/app/config/v1/config.go +++ b/internal/app/config/v1/config.go @@ -4,6 +4,216 @@ import ( "time" ) +// Deprecated configuration scheme +// version: +// server: +// http_addr: +// grpc_addr: +// debug_addr: +// grpc_connection_timeout: +// http_read_timeout: +// http_read_header_timeout: +// http_write_timeout: +// cors: +// allowed_origins: +// allowed_methods: +// allowed_headers: +// exposed_headers: +// allow_credentials: +// max_age: +// options_passthrough: +// jwt_secret_key: +// oidc: +// skip_verify: +// auth_urls: +// root_ca: +// ca_cert: +// private_key: +// ssl_skip_verify: +// allowed_clients: +// cache_secret_key: +// rate_limiters: +// : +// default: +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// spec_users: +// : +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// cache: +// inmemory: +// num_counters: +// max_cost: +// buffer_items: +// redis: +// addr: +// username: +// password: +// timeout: +// max_retries: +// min_retry_backoff: +// max_retry_backoff: +// db: +// name: +// host: +// port: +// pass: +// user: +// request_timeout: +// connection_pool_capacity: +// use_prepared_statements: +// clickhouse: +// addrs: +// database: +// username: +// password: +// sharded: +// dial_timeout: +// read_timeout: +// clients: +// seq_db_timeout: +// seq_db_avg_doc_size: +// seq_db_addrs: +// request_retries: +// initial_retry_backoff: +// max_retry_backoff: +// proxy_client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: +// seq_db: +// id: +// timeout: +// avg_doc_size: +// addrs: +// request_retries: +// initial_retry_backoff: +// max_retry_backoff: +// client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: +// handlers: +// seq_api: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// name: +// type: +// system_fields: +// name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// field: +// mode: +// values: +// process_fields: +// ignore_fields: +// envs: +// : +// seq_db_id: +// options: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// name: +// type: +// system_fields: +// name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// field: +// mode: +// values: +// process_fields: +// ignore_fields: +// default_env: +// error_groups: +// log_tags_mapping: +// env: +// service: +// release: +// query_filter: +// : +// mass_export: +// batch_size: +// workers_count: +// tasks_channel_size: +// part_length: +// url_prefix: +// allowed_users: +// file_store: +// s3: +// endpoint: +// access_key_id: +// secret_access_key: +// bucket_name: +// enable_ssl: +// session_store: +// redis: +// addr: +// username: +// password: +// timeout: +// max_retries: +// min_retry_backoff: +// max_retry_backoff: +// export_lifetime: +// seq_proxy_downloader: +// delay: +// initial_retry_backoff: +// max_retry_backoff: +// async_search: +// admin_users: +// list_query_length_limit: + type Config struct { Version *int `yaml:"version"` Server *Server `yaml:"server"` diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 882c00a..481625a 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -5,6 +5,212 @@ import ( "time" ) +// Actual configuration scheme +// version: +// server: +// http: +// addr: +// read_timeout: +// read_header_timeout: +// write_timeout: +// cors: +// allowed_origins: +// allowed_methods: +// allowed_headers: +// exposed_headers: +// allow_credentials: +// max_age: +// options_passthrough: +// grpc: +// addr: +// connection_timeout: +// debug: +// addr: +// auth: +// oidc: +// skip_verify: +// auth_urls: +// root_ca: +// ca_cert: +// private_key: +// ssl_skip_verify: +// allowed_clients: +// cache_secret_key: +// jwt: +// secret_key: +// rate_limiters: +// : +// default: +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// spec_users: +// : +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// clients: +// seq_db: +// id: +// timeout: +// avg_doc_size: +// addrs: +// request_retries: +// initial_retry_backoff: +// max_retry_backoff: +// client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: +// download_params: +// delay: +// initial_retry_backoff: +// max_retry_backoff: +// clickhouse: +// id: +// addrs: +// database: +// username: +// password: +// sharded: +// dial_timeout: +// read_timeout: +// handlers: +// seq_api: +// options: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// name: +// type: +// system_fields: +// name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// field: +// mode: +// values: +// process_fields: +// ignore_fields: +// envs: +// : +// seq_db_id: +// options: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// name: +// type: +// system_fields: +// name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// field: +// mode: +// values: +// process_fields: +// ignore_fields: +// default_env: +// error_groups: +// log_tags_mapping: +// env: +// service: +// release: +// query_filter: +// : +// mass_export: +// batch_size: +// workers_count: +// tasks_channel_size: +// part_length: +// url_prefix: +// allowed_users: +// file_store: +// s3: +// endpoint: +// access_key_id: +// secret_access_key: +// bucket_name: +// enable_ssl: +// session_store: +// redis_id: +// export_lifetime: +// download_params: +// delay: +// initial_retry_backoff: +// max_retry_backoff: +// async_search: +// admin_users: +// list_query_length_limit: +// db: +// name: +// host: +// port: +// pass: +// user: +// request_timeout: +// connection_pool_capacity: +// use_prepared_statements: +// cache: +// inmemory: +// id: +// num_counters: +// max_cost: +// buffer_items: +// redis: +// id: +// with_inmem_id: +// addr: +// username: +// password: +// timeout: +// max_retries: +// min_retry_backoff: +// max_retry_backoff: + const ( DefaultSeqDBClientID = "default" DefaultInmemCacheID = "seqapi" @@ -264,9 +470,9 @@ type Field struct { } type SeqAPI struct { - *SeqAPIOptions `yaml:",inline"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` + Options SeqAPIOptions `yaml:"options"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` } type SeqAPIEnv struct { @@ -419,14 +625,14 @@ func Normalize(cfg *Config) error { if r.ID == "" { return fmt.Errorf("redis cache ID cannot be empty") } - if _, ok := inmemIDs[r.ID]; ok { + if _, ok := redisIDs[r.ID]; ok { return fmt.Errorf("duplicate redis cache ID: %s", r.ID) } redisIDs[r.ID] = struct{}{} if r.WithInmemID != "" { - if _, ok := redisIDs[r.WithInmemID]; !ok { + if _, ok := inmemIDs[r.WithInmemID]; !ok { return fmt.Errorf("redis cache %q references unknown inmem cache id %q", r.ID, r.WithInmemID) } } @@ -441,7 +647,7 @@ func Normalize(cfg *Config) error { cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit } - setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + setSeqAPIOptionsDefaults(&cfg.Handlers.SeqAPI.Options) if len(cfg.Handlers.SeqAPI.Envs) > 0 { if cfg.Handlers.SeqAPI.DefaultEnv == "" { From b01ac86bb20dc521dda7401474f88ebfe185df67 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Thu, 23 Jul 2026 23:19:59 +0300 Subject: [PATCH 04/29] some refactor --- cmd/seq-ui/m.md | 0 cmd/seq-ui/main.go | 44 +++--- internal/api/seqapi/v1/http/aggregation.go | 4 +- .../api/seqapi/v1/http/aggregation_test.go | 14 +- .../api/seqapi/v1/http/aggregation_ts_test.go | 16 +-- internal/api/seqapi/v1/http/api.go | 126 +++++++----------- internal/api/seqapi/v1/http/events.go | 8 +- internal/api/seqapi/v1/seqapi.go | 2 +- internal/api/seqapi/v1/test/data.go | 20 +-- internal/app/config/config.go | 4 +- internal/app/config/migrate/v1_v2.go | 48 ++++--- internal/app/config/v2/config.go | 96 +++++++++---- internal/app/mw/auth.go | 2 +- internal/app/mw/ratelimiter.go | 2 +- internal/app/ratelimiter/ratelimiter.go | 2 +- internal/app/server/init.go | 30 ++--- internal/app/server/run.go | 14 +- internal/app/server/server.go | 2 +- internal/pkg/cache/ctor.go | 2 +- internal/pkg/cache/inmemory.go | 2 +- internal/pkg/cache/redis.go | 2 +- internal/pkg/mask/field_filter.go | 2 +- internal/pkg/mask/field_filter_test.go | 2 +- internal/pkg/mask/mask.go | 2 +- internal/pkg/mask/mask_test.go | 2 +- internal/pkg/mask/masker.go | 2 +- internal/pkg/mask/masker_test.go | 2 +- internal/pkg/redisclient/client.go | 2 +- .../pkg/service/async_searches/service.go | 2 +- internal/pkg/service/errorgroups/service.go | 2 +- .../pkg/service/errorgroups/service_test.go | 2 +- .../pkg/service/massexport/filestore/s3.go | 2 +- .../massexport/seq_proxy_downloader.go | 2 +- internal/pkg/service/massexport/service.go | 2 +- .../service/massexport/sessionstore/redis.go | 2 +- 35 files changed, 251 insertions(+), 217 deletions(-) delete mode 100644 cmd/seq-ui/m.md diff --git a/cmd/seq-ui/m.md b/cmd/seq-ui/m.md deleted file mode 100644 index e69de29..0000000 diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 6fa2344..8980870 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -23,6 +23,7 @@ import ( massexport_v1 "github.com/ozontech/seq-ui/internal/api/massexport/v1" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" + configloader "github.com/ozontech/seq-ui/internal/app/config" config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/server" "github.com/ozontech/seq-ui/internal/pkg/cache" @@ -74,7 +75,7 @@ func run(ctx context.Context) { logger.Warn("app uses the default config file, to provide your own config use -config flag") } - cfg, err := config.FromFile(*configPath) + cfg, err := configloader.FromFile(*configPath) if err != nil { logger.Fatal("read config file error", zap.Error(err)) } @@ -104,28 +105,21 @@ func run(ctx context.Context) { } } -func initApp(ctx context.Context, cfg config_v2.Config) *api.Registrar { +func initApp(ctx context.Context, cfg config.Config) *api.Registrar { logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { logger.Fatal("failed to init seq-db client", zap.Error(err)) } - defaultClientID := config.DefaultSeqDBClientID - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - defaultClientID = cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv].SeqDB - } - - defaultClient, exists := seqDBClients[defaultClientID] - if !exists { - logger.Fatal("default seq-db client not found", - zap.String("clientID", defaultClientID), - ) - } - var massExportV1 *massexport_v1.MassExport if cfg.Handlers.MassExport != nil { - exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, defaultClient) + massExportClient, ok := seqDBClients[cfg.Handlers.MassExport.SeqDBID] + if !ok { + logger.Fatal("mass_export seq_db client not found", zap.String("seq_db_id", cfg.Handlers.MassExport.SeqDBID)) + } + + exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, massExportClient) if err != nil { logger.Fatal("can't init export server", zap.Error(err)) } @@ -157,7 +151,7 @@ func initApp(ctx context.Context, cfg config_v2.Config) *api.Registrar { dashboardsV1 *dashboards_v1.Dashboards ) if db != nil { - repo := repository.New(db, cfg.Server.DB.RequestTimeout) + repo := repository.New(db, cfg.DB.RequestTimeout) userProfilesSvc := userprofile.New(repo.UserProfiles, repo.FavoriteQueries) dashboardsSvc := dashboards.New(repo.Dashboards) profiles.InitProfiles(repo.UserProfiles.GetOrCreate) @@ -165,7 +159,11 @@ func initApp(ctx context.Context, cfg config_v2.Config) *api.Registrar { userProfileV1 = userprofile_v1.New(userProfilesSvc) dashboardsV1 = dashboards_v1.New(dashboardsSvc) - asyncSearchesService = asyncsearches.New(ctx, repo, defaultClient, cfg.Handlers.AsyncSearch) + asyncSearchClient, ok := seqDBClients[cfg.Handlers.AsyncSearch.SeqDBID] + if !ok { + logger.Fatal("async_search seq_db client not found", zap.String("seq_db_id", cfg.Handlers.AsyncSearch.SeqDBID)) + } + asyncSearchesService = asyncsearches.New(ctx, repo, asyncSearchClient, cfg.Handlers.AsyncSearch) } seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService) @@ -201,7 +199,17 @@ func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb. } func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI config.SeqAPI) (seqdb.Client, error) { - clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * int(seqAPI.MaxSearchLimit) + maxSearchLimit := int(seqAPI.Options.MaxSearchLimit) + for _, env := range seqAPI.Envs { + if env.SeqDBID != cfg.ID { + continue + } + if l := int(env.Options.MaxSearchLimit); l > maxSearchLimit { + maxSearchLimit = l + } + } + + clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * maxSearchLimit if clientMaxRecvMsgSize < defaultClientMaxRecvMsgSize { clientMaxRecvMsgSize = defaultClientMaxRecvMsgSize } diff --git a/internal/api/seqapi/v1/http/aggregation.go b/internal/api/seqapi/v1/http/aggregation.go index b4ffebb..e423159 100644 --- a/internal/api/seqapi/v1/http/aggregation.go +++ b/internal/api/seqapi/v1/http/aggregation.go @@ -94,7 +94,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { getAggResp := getAggregationResponseFromProto(resp) - if params.masker != nil { + if a.globalParams.masker != nil { buf := make([]string, 0) for i, agg := range getAggResp.Aggregations { buf = buf[:0] @@ -108,7 +108,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { field = aggReq.GroupBy } - buf = params.masker.MaskAgg(field, buf) + buf = a.globalParams.masker.MaskAgg(field, buf) for j, key := range buf { getAggResp.Aggregations[i].Buckets[j].Key = key diff --git a/internal/api/seqapi/v1/http/aggregation_test.go b/internal/api/seqapi/v1/http/aggregation_test.go index 6a675f4..6b75c75 100644 --- a/internal/api/seqapi/v1/http/aggregation_test.go +++ b/internal/api/seqapi/v1/http/aggregation_test.go @@ -11,7 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -63,7 +63,7 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: false, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -106,7 +106,7 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: false, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -168,7 +168,7 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: false, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -203,7 +203,7 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: true, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -218,7 +218,7 @@ func TestServeGetAggregation(t *testing.T) { }, wantErr: true, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 2, }, }, @@ -242,7 +242,7 @@ func TestServeGetAggregation(t *testing.T) { }, wantErr: true, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, diff --git a/internal/api/seqapi/v1/http/aggregation_ts_test.go b/internal/api/seqapi/v1/http/aggregation_ts_test.go index 9c4f86c..cd06b34 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts_test.go +++ b/internal/api/seqapi/v1/http/aggregation_ts_test.go @@ -17,7 +17,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -106,7 +106,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"test_count1":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count1":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count1":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}},{"data":{"result":[{"metric":{"test_count2":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count2":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count2":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 100, }, @@ -162,7 +162,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"test_count1":"test1"},"values":[{"timestamp":1695637231,"value":3}]},{"metric":{"test_count1":"test2"},"values":[{"timestamp":1695637232,"value":6}]},{"metric":{"test_count1":"test3"},"values":[{"timestamp":1695637233,"value":9}]}]}},{"data":{"result":[{"metric":{"test_count2":"test1"},"values":[{"timestamp":1695637231,"value":2}]},{"metric":{"test_count2":"test2"},"values":[{"timestamp":1695637232,"value":4}]},{"metric":{"test_count2":"test3"},"values":[{"timestamp":1695637233,"value":6}]}],"target_bucket_rate":"2s"}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 100, }, @@ -221,7 +221,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"quantile":"p95","service":"test1"},"values":[{"timestamp":1695637231,"value":100}]},{"metric":{"quantile":"p99","service":"test1"},"values":[{"timestamp":1695637231,"value":150}]},{"metric":{"quantile":"p95","service":"test2"},"values":[{"timestamp":1695637232,"value":100}]},{"metric":{"quantile":"p99","service":"test2"},"values":[{"timestamp":1695637232,"value":150}]},{"metric":{"quantile":"p95","service":"test3"},"values":[{"timestamp":1695637233,"value":100}]},{"metric":{"quantile":"p99","service":"test3"},"values":[{"timestamp":1695637233,"value":150}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 100, }, @@ -247,7 +247,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":null,"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 100, }, @@ -263,7 +263,7 @@ func TestServeGetAggregationTs(t *testing.T) { reqBody: formatReqBody(aggregationTsQueries{{}, {}, {}}), wantStatus: http.StatusBadRequest, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 2, MaxBucketsPerAggregationTs: 100, }, @@ -278,7 +278,7 @@ func TestServeGetAggregationTs(t *testing.T) { }), wantStatus: http.StatusBadRequest, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 1, }, @@ -297,7 +297,7 @@ func TestServeGetAggregationTs(t *testing.T) { }, wantStatus: http.StatusInternalServerError, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index c762883..bb567fc 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -10,7 +10,7 @@ import ( "github.com/gofrs/uuid" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/tokenlimiter" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" @@ -24,115 +24,89 @@ import ( type apiParams struct { client seqdb.Client options *config.SeqAPIOptions - fieldsCache *fieldsCache - masker *mask.Masker - pinnedFields fields - systemFields fields exportLimiter *tokenlimiter.Limiter } +type apiGlobalParams struct { + fieldsCache *fieldsCache + masker *mask.Masker + pinnedFields fields + systemFields fields +} + type API struct { - config config.SeqAPI - params apiParams - paramsByEnv map[string]apiParams - inmemWithRedisCache cache.Cache - redisCache cache.Cache - nowFn func() time.Time - asyncSearches asyncsearches.Service - envsResponse getEnvsResponse + config config.SeqAPI + globalParams apiGlobalParams + paramsByEnv map[string]apiParams + cache cache.Cache + nowFn func() time.Time + asyncSearches asyncsearches.Service + envsResponse getEnvsResponse } func New( cfg config.SeqAPI, seqDBСlients map[string]seqdb.Client, - inmemWithRedisCache cache.Cache, - redisCache cache.Cache, + cache cache.Cache, asyncSearches asyncsearches.Service, ) *API { - var globalfCache *fieldsCache - if cfg.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.FieldsCacheTTL) + globalParams := apiGlobalParams{ + pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), + systemFields: parseFields(cfg.GlobalOptions.SystemFields), + } + if cfg.GlobalOptions.FieldsCacheTTL > 0 { + globalParams.fieldsCache = newFieldsCache(cfg.GlobalOptions.FieldsCacheTTL) } - globalMasker, err := mask.New(cfg.Masking) + globalMasker, err := mask.New(cfg.GlobalOptions.Masking) if err != nil { logger.Fatal("failed to init masking", zap.Error(err)) } - globalPinnedFields := parseFields(cfg.PinnedFields) - globalSystemFields := parseFields(cfg.SystemFields) - globalExportLimiter := tokenlimiter.New(cfg.MaxParallelExportRequests) - - var params apiParams - var paramsByEnv map[string]apiParams + globalParams.masker = globalMasker + globalExportLimiter := tokenlimiter.New(cfg.Options.MaxParallelExportRequests) + paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { - paramsByEnv = make(map[string]apiParams) for envName, envConfig := range cfg.Envs { - client := seqDBСlients[envConfig.SeqDB] - options := envConfig.Options - - var envfCache *fieldsCache - if options.FieldsCacheTTL > 0 { - envfCache = newFieldsCache(options.FieldsCacheTTL) - } - - var envMasker, err = mask.New(options.Masking) - if err != nil { - logger.Fatal("failed to init env masking", zap.Error(err)) - } - - envPinnedFields := parseFields(options.PinnedFields) - envSystemFields := parseFields(options.SystemFields) - envExportLimiter := tokenlimiter.New(options.MaxParallelExportRequests) - paramsByEnv[envName] = apiParams{ - client: client, - options: options, - fieldsCache: envfCache, - masker: envMasker, - pinnedFields: envPinnedFields, - systemFields: envSystemFields, - exportLimiter: envExportLimiter, + client: seqDBСlients[envConfig.SeqDBID], + options: envConfig.Options, + exportLimiter: tokenlimiter.New(envConfig.Options.MaxParallelExportRequests), } } } else { - client, exists := seqDBСlients[config.DefaultSeqDBClientID] - if !exists { - logger.Fatal("default client not found", - zap.String("clientID", config.DefaultSeqDBClientID)) + if len(seqDBСlients) != 1 { + logger.Fatal("seq_api.envs is empty, configure envs when clients.seq_db has more than one instance") + } + + var client seqdb.Client + for _, c := range seqDBСlients { + client = c } - params = apiParams{ + paramsByEnv[""] = apiParams{ client: client, - options: cfg.SeqAPIOptions, - fieldsCache: globalfCache, - masker: globalMasker, - pinnedFields: globalPinnedFields, - systemFields: globalSystemFields, + options: &cfg.Options, exportLimiter: globalExportLimiter, } } + // for export - if len(cfg.Envs) > 0 { + if globalParams.masker != nil { for _, param := range paramsByEnv { - if param.masker != nil { - param.client.WithMasking(param.masker) - } + param.client.WithMasking(globalParams.masker) } - } else if params.masker != nil { - params.client.WithMasking(params.masker) } return &API{ - config: cfg, - params: params, - paramsByEnv: paramsByEnv, - inmemWithRedisCache: inmemWithRedisCache, - redisCache: redisCache, - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + config: cfg, + globalParams: globalParams, + paramsByEnv: paramsByEnv, + cache: cache, + nowFn: time.Now, + asyncSearches: asyncSearches, + envsResponse: parseEnvs(cfg), } } @@ -219,7 +193,7 @@ func parseEnvs(cfg config.SeqAPI) getEnvsResponse { envs = append(envs, createEnvInfo(name, envConfig.Options)) } } else { - envs = []envInfo{createEnvInfo("", cfg.SeqAPIOptions)} + envs = []envInfo{createEnvInfo("", &cfg.Options)} } return getEnvsResponse{Envs: envs} } @@ -236,7 +210,7 @@ const ( func (a *API) GetEnvParams(env string) (apiParams, error) { if len(a.config.Envs) == 0 { - return a.params, nil + return a.paramsByEnv[""], nil } if env == "" { diff --git a/internal/api/seqapi/v1/http/events.go b/internal/api/seqapi/v1/http/events.go index 006293a..6f8154a 100644 --- a/internal/api/seqapi/v1/http/events.go +++ b/internal/api/seqapi/v1/http/events.go @@ -57,8 +57,8 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { if cached, err := a.inmemWithRedisCache.Get(ctx, id); err == nil { e := &seqapi.Event{} if err = proto.Unmarshal([]byte(cached), e); err == nil { - if params.masker != nil { - params.masker.Mask(e.Data) + if a.globalParams.masker != nil { + a.globalParams.masker.Mask(e.Data) } wr.WriteJson(getEventResponseFromProto(&seqapi.GetEventResponse{Event: e})) return @@ -74,8 +74,8 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { return } - if params.masker != nil && resp.Event != nil { - params.masker.Mask(resp.Event.Data) + if a.globalParams.masker != nil && resp.Event != nil { + a.globalParams.masker.Mask(resp.Event.Data) } if data, err := proto.Marshal(resp.Event); err == nil { diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index f1a08c4..f32d657 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -5,7 +5,7 @@ import ( grpc_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/grpc" http_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/http" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" diff --git a/internal/api/seqapi/v1/test/data.go b/internal/api/seqapi/v1/test/data.go index 0ab1d44..b32962f 100644 --- a/internal/api/seqapi/v1/test/data.go +++ b/internal/api/seqapi/v1/test/data.go @@ -6,7 +6,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" mock_asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches/mock" @@ -111,22 +111,22 @@ func MakeAggregations(aggCount, bucketCount int, opts *MakeAggOpts) []*seqapi.Ag } func SetCfgDefaults(cfg config.SeqAPI) config.SeqAPI { - if cfg.MaxAggregationsPerRequest <= 0 { - cfg.MaxAggregationsPerRequest = 1 + if cfg.Options.MaxAggregationsPerRequest <= 0 { + cfg.Options.MaxAggregationsPerRequest = 1 } - if cfg.MaxParallelExportRequests <= 0 { - cfg.MaxParallelExportRequests = 1 + if cfg.Options.MaxParallelExportRequests <= 0 { + cfg.Options.MaxParallelExportRequests = 1 } - if cfg.MaxSearchTotalLimit <= 0 { - cfg.MaxSearchTotalLimit = 1000000 + if cfg.Options.MaxSearchTotalLimit <= 0 { + cfg.Options.MaxSearchTotalLimit = 1000000 } - if cfg.MaxSearchOffsetLimit <= 0 { - cfg.MaxSearchOffsetLimit = 1000000 + if cfg.Options.MaxSearchOffsetLimit <= 0 { + cfg.Options.MaxSearchOffsetLimit = 1000000 } for envName, envConfig := range cfg.Envs { if envConfig.Options == nil { - envConfig.Options = cfg.SeqAPIOptions + envConfig.Options = &cfg.Options cfg.Envs[envName] = envConfig } } diff --git a/internal/app/config/config.go b/internal/app/config/config.go index 9f46c43..000e69d 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -12,8 +12,6 @@ import ( ) const ( - configV1Name = "config_v1" - configV2Name = "config_v2" previousVersion = 1 currentVersion = 2 ) @@ -52,7 +50,7 @@ func FromFile(cfgPath string) (v2.Config, error) { } if err := v2.Normalize(&cfg); err != nil { - return v2.Config{}, fmt.Errorf("normalize config: %w") + return v2.Config{}, fmt.Errorf("normalize config: %w", err) } return cfg, nil diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index 522fc50..f0b2bc4 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -250,11 +250,17 @@ func migrateHandlers(src *v1.Handlers, cfg *v2.Config) *v2.Handlers { } func migrateSeqAPI(src v1.SeqAPI) v2.SeqAPI { - return v2.SeqAPI{ - SeqAPIOptions: migrateSeqAPIOptions(src.SeqAPIOptions), - Envs: migrateSeqAPIEnvs(src.Envs), - DefaultEnv: src.DefaultEnv, + dst := v2.SeqAPI{ + Envs: migrateSeqAPIEnvs(src.Envs), + DefaultEnv: src.DefaultEnv, } + + if src.SeqAPIOptions != nil { + dst.GlobalOptions = migrateSeqAPIOptionsToGlobal(src.SeqAPIOptions) + dst.Options = migrateSeqAPIOptions(src.SeqAPIOptions) + } + + return dst } func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv) map[string]v2.SeqAPIEnv { @@ -264,21 +270,34 @@ func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv) map[string]v2.SeqAPIEnv { dst := make(map[string]v2.SeqAPIEnv, len(envs)) for name, cfg := range envs { - dst[name] = v2.SeqAPIEnv{ - SeqDB: cfg.SeqDB, - Options: migrateSeqAPIOptions(cfg.Options), + env := v2.SeqAPIEnv{ + SeqDBID: cfg.SeqDB, } + if cfg.Options != nil { + envOptions := migrateSeqAPIOptions(cfg.Options) + env.Options = &envOptions + } + + dst[name] = env } return dst } -func migrateSeqAPIOptions(options *v1.SeqAPIOptions) *v2.SeqAPIOptions { - if options == nil { - return nil +func migrateSeqAPIOptionsToGlobal(options *v1.SeqAPIOptions) v2.SeqAPIGlobalOptions { + return v2.SeqAPIGlobalOptions{ + EventsCacheTTL: options.EventsCacheTTL, + PinnedFields: migrateFields(options.PinnedFields), + SystemFields: migrateFields(options.SystemFields), + LogsLifespanCacheKey: options.LogsLifespanCacheKey, + LogsLifespanCacheTTL: options.LogsLifespanCacheTTL, + FieldsCacheTTL: options.FieldsCacheTTL, + Masking: migrateMasking(options.Masking), } +} - return &v2.SeqAPIOptions{ +func migrateSeqAPIOptions(options *v1.SeqAPIOptions) v2.SeqAPIOptions { + return v2.SeqAPIOptions{ MaxSearchLimit: options.MaxSearchLimit, MaxSearchTotalLimit: options.MaxSearchTotalLimit, MaxSearchOffsetLimit: options.MaxSearchOffsetLimit, @@ -287,13 +306,6 @@ func migrateSeqAPIOptions(options *v1.SeqAPIOptions) *v2.SeqAPIOptions { MaxParallelExportRequests: options.MaxParallelExportRequests, MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, - EventsCacheTTL: options.EventsCacheTTL, - PinnedFields: migrateFields(options.PinnedFields), - SystemFields: migrateFields(options.SystemFields), - LogsLifespanCacheKey: options.LogsLifespanCacheKey, - LogsLifespanCacheTTL: options.LogsLifespanCacheTTL, - FieldsCacheTTL: options.FieldsCacheTTL, - Masking: migrateMasking(options.Masking), } } diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 481625a..9350474 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -368,6 +368,7 @@ type FileStore struct { } type MassExport struct { + SeqDBID string `yaml:"seq_db_id"` BatchSize uint64 `yaml:"batch_size"` WorkersCount int `yaml:"workers_count"` TasksChannelSize int `yaml:"tasks_channel_size"` @@ -470,32 +471,37 @@ type Field struct { } type SeqAPI struct { - Options SeqAPIOptions `yaml:"options"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` + CacheID string `yaml:"cache_id"` + GlobalOptions SeqAPIGlobalOptions `yaml:"global_options"` + Options SeqAPIOptions `yaml:"options"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type SeqAPIGlobalOptions struct { + PinnedFields []Field `yaml:"pinned_fields"` + SystemFields []Field `yaml:"system_fields"` + Masking *Masking `yaml:"masking"` + LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` + EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` + LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` + FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` } type SeqAPIEnv struct { - SeqDB string `yaml:"seq_db_id"` + SeqDBID string `yaml:"seq_db_id"` Options *SeqAPIOptions `yaml:"options"` } type SeqAPIOptions struct { - MaxSearchLimit int32 `yaml:"max_search_limit"` - MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` - MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` - MaxExportLimit int32 `yaml:"max_export_limit"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` - EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` - PinnedFields []Field `yaml:"pinned_fields"` - SystemFields []Field `yaml:"system_fields"` - LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` - LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` - FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` - Masking *Masking `yaml:"masking"` + MaxSearchLimit int32 `yaml:"max_search_limit"` + MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` + MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` + MaxExportLimit int32 `yaml:"max_export_limit"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` } type Masking struct { @@ -534,11 +540,13 @@ type LogTagsMapping struct { } type ErrorGroups struct { + CHID string `yaml:"ch_id"` LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` QueryFilter map[string]string `yaml:"query_filter"` } type AsyncSearch struct { + SeqDBID string `yaml:"seq_db_id"` AdminUsers []string `yaml:"admin_users"` ListQueryLengthLimit int `yaml:"list_query_length_limit"` } @@ -647,6 +655,7 @@ func Normalize(cfg *Config) error { cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit } + setSeqAPIGlobalOptionsDefaults(&cfg.Handlers.SeqAPI.GlobalOptions) setSeqAPIOptionsDefaults(&cfg.Handlers.SeqAPI.Options) if len(cfg.Handlers.SeqAPI.Envs) > 0 { @@ -659,16 +668,12 @@ func Normalize(cfg *Config) error { } for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if _, ok := seqDBIDs[envConfig.SeqDB]; !ok { - return fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) - } - - if envConfig.Options == nil { - envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions - } else { - setSeqAPIOptionsDefaults(envConfig.Options) + if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + return fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDBID, envName) } + merged := mergeSeqAPIOptions(cfg.Handlers.SeqAPI.Options, envConfig.Options) + envConfig.Options = &merged cfg.Handlers.SeqAPI.Envs[envName] = envConfig } } @@ -704,6 +709,43 @@ func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { if options.MaxExportLimit <= 0 { options.MaxExportLimit = defaultMaxExportLimit } +} + +func mergeSeqAPIOptions(global SeqAPIOptions, envOptions *SeqAPIOptions) SeqAPIOptions { + merged := global + if envOptions == nil { + return merged + } + + if envOptions.MaxAggregationsPerRequest > 0 { + merged.MaxAggregationsPerRequest = envOptions.MaxAggregationsPerRequest + } + if envOptions.MaxBucketsPerAggregationTs > 0 { + merged.MaxBucketsPerAggregationTs = envOptions.MaxBucketsPerAggregationTs + } + if envOptions.MaxParallelExportRequests > 0 { + merged.MaxParallelExportRequests = envOptions.MaxParallelExportRequests + } + if envOptions.MaxSearchLimit > 0 { + merged.MaxSearchLimit = envOptions.MaxSearchLimit + } + if envOptions.MaxSearchTotalLimit > 0 { + merged.MaxSearchTotalLimit = envOptions.MaxSearchTotalLimit + } + if envOptions.MaxSearchOffsetLimit > 0 { + merged.MaxSearchOffsetLimit = envOptions.MaxSearchOffsetLimit + } + if envOptions.MaxExportLimit > 0 { + merged.MaxExportLimit = envOptions.MaxExportLimit + } + if envOptions.SeqCLIMaxSearchLimit > 0 { + merged.SeqCLIMaxSearchLimit = envOptions.SeqCLIMaxSearchLimit + } + + return merged +} + +func setSeqAPIGlobalOptionsDefaults(options *SeqAPIGlobalOptions) { if options.EventsCacheTTL <= 0 { options.EventsCacheTTL = defaultEventsCacheTTL } diff --git a/internal/app/mw/auth.go b/internal/app/mw/auth.go index a927e2d..ef8ee74 100644 --- a/internal/app/mw/auth.go +++ b/internal/app/mw/auth.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/ozontech/seq-ui/internal/app/auth" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/logger" ) diff --git a/internal/app/mw/ratelimiter.go b/internal/app/mw/ratelimiter.go index f181616..ccee177 100644 --- a/internal/app/mw/ratelimiter.go +++ b/internal/app/mw/ratelimiter.go @@ -7,7 +7,7 @@ import ( "net/http" "strconv" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/ratelimiter" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/metric" diff --git a/internal/app/ratelimiter/ratelimiter.go b/internal/app/ratelimiter/ratelimiter.go index dadee28..096ac91 100644 --- a/internal/app/ratelimiter/ratelimiter.go +++ b/internal/app/ratelimiter/ratelimiter.go @@ -8,7 +8,7 @@ import ( "github.com/throttled/throttled/v2" "github.com/throttled/throttled/v2/store/memstore" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) const rateLimiterQuantityPerCall = 1 diff --git a/internal/app/server/init.go b/internal/app/server/init.go index 26d31a7..affc9c8 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -30,7 +30,7 @@ var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { var err error - s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.JWTSecretKey, s.config.OIDC, s.config.Cache) + s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.Auth.JWT.SecretKey, s.config.Auth.OIDC, s.Cache) if err != nil { return err } @@ -54,22 +54,22 @@ func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { // setupCORS applies CORS policies set in config to the provided mux. func (s *Server) setupCORS(mux *chi.Mux) { - allowedOrigins := s.config.CORS.AllowedOrigins + allowedOrigins := s.config.HTTP.CORS.AllowedOrigins if len(allowedOrigins) == 0 { allowedOrigins = append(allowedOrigins, defaultCORSAllowedOrigins) } - allowedMethods := s.config.CORS.AllowedMethods + allowedMethods := s.config.HTTP.CORS.AllowedMethods if len(allowedMethods) == 0 { allowedMethods = append(allowedMethods, defaultCORSAllowedMethods...) } mux.Use(cors.Handler(cors.Options{ AllowedOrigins: allowedOrigins, AllowedMethods: allowedMethods, - AllowedHeaders: s.config.CORS.AllowedHeaders, - ExposedHeaders: s.config.CORS.ExposedHeaders, - AllowCredentials: s.config.CORS.AllowCredentials, - MaxAge: s.config.CORS.MaxAge, - OptionsPassthrough: s.config.CORS.OptionsPassthrough, + AllowedHeaders: s.config.HTTP.CORS.AllowedHeaders, + ExposedHeaders: s.config.HTTP.CORS.ExposedHeaders, + AllowCredentials: s.config.HTTP.CORS.AllowCredentials, + MaxAge: s.config.HTTP.CORS.MaxAge, + OptionsPassthrough: s.config.HTTP.CORS.OptionsPassthrough, })) } @@ -130,7 +130,7 @@ func (s *Server) prepareGRPCServer(registrar *api.Registrar) { opts := []grpc.ServerOption{ grpc.UnaryInterceptor(grpc_mw.ChainUnaryServer(interceptors...)), - grpc.ConnectionTimeout(s.config.GRPCConnectionTimeout), + grpc.ConnectionTimeout(s.config.GRPC.ConnectionTimeout), } s.grpcServer = grpc.NewServer(opts...) @@ -143,7 +143,7 @@ func (s *Server) prepareGRPCServer(registrar *api.Registrar) { // prepareHTTPServer prepares HTTP server with applied CORS policies and added interceptors. func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar) { mux := chi.NewMux() - if s.config.CORS != nil { + if s.config.HTTP.CORS != nil { s.setupCORS(mux) } interceptors := chi.Middlewares{ @@ -172,7 +172,7 @@ func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar // prepareHTTPMux prepares debug HTTP server mux with applied CORS policies and added interceptors. func (s *Server) prepareDebugServer(ctx context.Context) error { mux := chi.NewMux() - if s.config.CORS != nil { + if s.config.HTTP.CORS != nil { s.setupCORS(mux) } interceptors := chi.Middlewares{ @@ -181,7 +181,7 @@ func (s *Server) prepareDebugServer(ctx context.Context) error { mux.Use(interceptors...) mux.Handle("/metrics", promhttp.Handler()) serveHealth(mux) - _, port, err := net.SplitHostPort(s.config.HTTPAddr) + _, port, err := net.SplitHostPort(s.config.HTTP.Addr) if err != nil { return err } @@ -195,9 +195,9 @@ func (s *Server) prepareDebugServer(ctx context.Context) error { func (s *Server) makeHTTPServer(ctx context.Context, mux *chi.Mux) *http.Server { return &http.Server{ Handler: mux, - ReadHeaderTimeout: s.config.HTTPReadHeaderTimeout, - ReadTimeout: s.config.HTTPReadTimeout, - WriteTimeout: s.config.HTTPWriteTimeout, + ReadHeaderTimeout: s.config.HTTP.ReadHeaderTimeout, + ReadTimeout: s.config.HTTP.ReadTimeout, + WriteTimeout: s.config.HTTP.WriteTimeout, MaxHeaderBytes: maxHTTPHeaderBytes, // 4 KiB BaseContext: func(_ net.Listener) context.Context { return ctx diff --git a/internal/app/server/run.go b/internal/app/server/run.go index ce7dc2c..a7346ba 100644 --- a/internal/app/server/run.go +++ b/internal/app/server/run.go @@ -18,7 +18,7 @@ func (s *Server) Run(ctx context.Context) error { // run gRPC server errWg.Go(func() error { - lis, err := net.Listen("tcp", s.config.GRPCAddr) + lis, err := net.Listen("tcp", s.config.GRPC.Addr) if err != nil { return err } @@ -28,7 +28,7 @@ func (s *Server) Run(ctx context.Context) error { // run HTTP server errWg.Go(func() error { - l, err := net.Listen("tcp", s.config.HTTPAddr) + l, err := net.Listen("tcp", s.config.HTTP.Addr) if err != nil { return err } @@ -38,7 +38,7 @@ func (s *Server) Run(ctx context.Context) error { // run debug server errWg.Go(func() error { - l, err := net.Listen("tcp", s.config.DebugAddr) + l, err := net.Listen("tcp", s.config.Debug.Addr) if err != nil { return err } @@ -47,12 +47,12 @@ func (s *Server) Run(ctx context.Context) error { }) errWg.Go(func() error { - swaggerAddr := s.config.DebugAddr + swaggerUIPrefix + swaggerAddr := s.config.Debug.Addr + swaggerUIPrefix logger.Info("app started", - zap.String("http", s.config.HTTPAddr), - zap.String("debug", s.config.DebugAddr), + zap.String("http", s.config.HTTP.Addr), + zap.String("debug", s.config.Debug.Addr), zap.String("swagger", swaggerAddr), - zap.String("grpc", s.config.GRPCAddr), + zap.String("grpc", s.config.GRPC.Addr), ) return nil }) diff --git a/internal/app/server/server.go b/internal/app/server/server.go index 6872959..3d9e256 100644 --- a/internal/app/server/server.go +++ b/internal/app/server/server.go @@ -8,7 +8,7 @@ import ( "google.golang.org/grpc" "github.com/ozontech/seq-ui/internal/api" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/mw" ) diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index b14bc3a..b9030c0 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -6,7 +6,7 @@ import ( "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/logger" ) diff --git a/internal/pkg/cache/inmemory.go b/internal/pkg/cache/inmemory.go index acf69cf..1236b87 100644 --- a/internal/pkg/cache/inmemory.go +++ b/internal/pkg/cache/inmemory.go @@ -7,7 +7,7 @@ import ( "github.com/dgraph-io/ristretto" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/metric" ) diff --git a/internal/pkg/cache/redis.go b/internal/pkg/cache/redis.go index 4e60d44..a36782c 100644 --- a/internal/pkg/cache/redis.go +++ b/internal/pkg/cache/redis.go @@ -8,7 +8,7 @@ import ( "github.com/redis/go-redis/v9" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/redisclient" "github.com/ozontech/seq-ui/metric" ) diff --git a/internal/pkg/mask/field_filter.go b/internal/pkg/mask/field_filter.go index 2eba3dc..3f108ba 100644 --- a/internal/pkg/mask/field_filter.go +++ b/internal/pkg/mask/field_filter.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) type filterCondition int8 diff --git a/internal/pkg/mask/field_filter_test.go b/internal/pkg/mask/field_filter_test.go index 4c368bc..fe69ba7 100644 --- a/internal/pkg/mask/field_filter_test.go +++ b/internal/pkg/mask/field_filter_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestFieldFilterSetCreate(t *testing.T) { diff --git a/internal/pkg/mask/mask.go b/internal/pkg/mask/mask.go index f3d4000..c16035f 100644 --- a/internal/pkg/mask/mask.go +++ b/internal/pkg/mask/mask.go @@ -7,7 +7,7 @@ import ( "slices" "strings" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) const maskSymbol = byte('*') diff --git a/internal/pkg/mask/mask_test.go b/internal/pkg/mask/mask_test.go index afe054e..612d237 100644 --- a/internal/pkg/mask/mask_test.go +++ b/internal/pkg/mask/mask_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestMaskCompile(t *testing.T) { diff --git a/internal/pkg/mask/masker.go b/internal/pkg/mask/masker.go index 90adb62..7d75b26 100644 --- a/internal/pkg/mask/masker.go +++ b/internal/pkg/mask/masker.go @@ -3,7 +3,7 @@ package mask import ( "fmt" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) type Masker struct { diff --git a/internal/pkg/mask/masker_test.go b/internal/pkg/mask/masker_test.go index b8da9d2..3150786 100644 --- a/internal/pkg/mask/masker_test.go +++ b/internal/pkg/mask/masker_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestMaskerMask(t *testing.T) { diff --git a/internal/pkg/redisclient/client.go b/internal/pkg/redisclient/client.go index e9a879b..00f5d7a 100644 --- a/internal/pkg/redisclient/client.go +++ b/internal/pkg/redisclient/client.go @@ -6,7 +6,7 @@ import ( "github.com/redis/go-redis/v9" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func New(ctx context.Context, cfg *config.Redis) (*redis.Client, error) { diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index d6f4825..1c55396 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -10,7 +10,7 @@ import ( "github.com/cenkalti/backoff/v4" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/repository" diff --git a/internal/pkg/service/errorgroups/service.go b/internal/pkg/service/errorgroups/service.go index f6af295..b537bde 100644 --- a/internal/pkg/service/errorgroups/service.go +++ b/internal/pkg/service/errorgroups/service.go @@ -8,7 +8,7 @@ import ( "golang.org/x/sync/errgroup" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" repositorych "github.com/ozontech/seq-ui/internal/pkg/repository_ch" ) diff --git a/internal/pkg/service/errorgroups/service_test.go b/internal/pkg/service/errorgroups/service_test.go index be857e9..49cc37d 100644 --- a/internal/pkg/service/errorgroups/service_test.go +++ b/internal/pkg/service/errorgroups/service_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" mock "github.com/ozontech/seq-ui/internal/pkg/repository_ch/mock" ) diff --git a/internal/pkg/service/massexport/filestore/s3.go b/internal/pkg/service/massexport/filestore/s3.go index 469e465..2903e6e 100644 --- a/internal/pkg/service/massexport/filestore/s3.go +++ b/internal/pkg/service/massexport/filestore/s3.go @@ -11,7 +11,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) type s3FileStore struct { diff --git a/internal/pkg/service/massexport/seq_proxy_downloader.go b/internal/pkg/service/massexport/seq_proxy_downloader.go index 3ff9163..4a92f3e 100644 --- a/internal/pkg/service/massexport/seq_proxy_downloader.go +++ b/internal/pkg/service/massexport/seq_proxy_downloader.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/metric" diff --git a/internal/pkg/service/massexport/service.go b/internal/pkg/service/massexport/service.go index 3c21309..e65ba6b 100644 --- a/internal/pkg/service/massexport/service.go +++ b/internal/pkg/service/massexport/service.go @@ -7,7 +7,7 @@ import ( "slices" "time" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/service/massexport/filestore" diff --git a/internal/pkg/service/massexport/sessionstore/redis.go b/internal/pkg/service/massexport/sessionstore/redis.go index f24d9b1..e5a7988 100644 --- a/internal/pkg/service/massexport/sessionstore/redis.go +++ b/internal/pkg/service/massexport/sessionstore/redis.go @@ -11,7 +11,7 @@ import ( "github.com/redis/go-redis/v9" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/redisclient" "github.com/ozontech/seq-ui/logger" From 1eb14b824ef4f25a57cc8a888a47f423608df473 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Tue, 28 Jul 2026 17:17:44 +0300 Subject: [PATCH 05/29] update --- cmd/config-migrate/main.go | 3 ++- cmd/seq-ui/main.go | 27 ++++++++------------------- internal/app/config/config.go | 3 ++- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go index 08fd1b8..3dd1f2c 100644 --- a/cmd/config-migrate/main.go +++ b/cmd/config-migrate/main.go @@ -5,9 +5,10 @@ import ( "flag" "os" + "go.uber.org/zap" + "github.com/ozontech/seq-ui/internal/app/config" "github.com/ozontech/seq-ui/logger" - "go.uber.org/zap" ) var ( diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 8980870..c35fc71 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -111,14 +111,12 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { if err != nil { logger.Fatal("failed to init seq-db client", zap.Error(err)) } - var massExportV1 *massexport_v1.MassExport if cfg.Handlers.MassExport != nil { massExportClient, ok := seqDBClients[cfg.Handlers.MassExport.SeqDBID] if !ok { logger.Fatal("mass_export seq_db client not found", zap.String("seq_db_id", cfg.Handlers.MassExport.SeqDBID)) } - exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, massExportClient) if err != nil { logger.Fatal("can't init export server", zap.Error(err)) @@ -126,17 +124,10 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { massExportV1 = massexport_v1.New(exportServer) } - - logger.Info("initializing inmemory with redis seqapi cache") - inmemWithRedisCache, err := cache.NewInmemoryWithRedisOrInmemory(ctx, cfg.Server.Cache) - if err != nil { - logger.Fatal("failed to init inmemory with redis seqapi cache", zap.Error(err)) - } - - logger.Info("initializing redis seqapi cache") - redisCache, err := cache.NewRedisOrInmemory(ctx, cfg.Server.Cache) + logger.Info("initializing caches") + caches, err := cache.FromConfig(ctx, cfg.Cache) if err != nil { - logger.Fatal("failed to init redis seqapi cache", zap.Error(err)) + logger.Fatal("failed to init caches", zap.Error(err)) } logger.Info("initializing db") @@ -166,7 +157,10 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { asyncSearchesService = asyncsearches.New(ctx, repo, asyncSearchClient, cfg.Handlers.AsyncSearch) } - seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService) + seqApiCache := caches[cfg.Handlers.SeqAPI.CacheID] + seqApiRedisCache := caches[cfg.Handlers.SeqAPI.RedisID] + + seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesService) logger.Info("initializing clickhouse") ch, err := initClickHouse(ctx, cfg.Server.CH) @@ -176,7 +170,7 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { var errorGroupsV1 *errorgroups_v1.ErrorGroups if ch != nil { - repo := repositorych.New(ch, cfg.Server.CH.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) + repo := repositorych.New(ch, cfg.Clients.ClickHouse.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) svc := errorgroups.New(repo, cfg.Handlers.ErrorGroups.LogTagsMapping) errorGroupsV1 = errorgroups_v1.New(svc) @@ -208,12 +202,10 @@ func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI confi maxSearchLimit = l } } - clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * maxSearchLimit if clientMaxRecvMsgSize < defaultClientMaxRecvMsgSize { clientMaxRecvMsgSize = defaultClientMaxRecvMsgSize } - clientParams := seqdb.ClientParams{ Addrs: cfg.Addrs, Timeout: cfg.Timeout, @@ -232,18 +224,15 @@ func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI confi return seqdb.NewGRPCClient(ctx, clientParams) } - func initDb(ctx context.Context, cfg *config_v2.DB) (*pgxpool.Pool, error) { if cfg == nil { logger.Warn("db config is nil, running without db") return nil, nil } - pgxCfg, err := pgxpool.ParseConfig(cfg.ConnString()) if err != nil { return nil, fmt.Errorf("can't parse connection string: %w", err) } - if !*cfg.UsePreparedStatements { // By default, pgx uses the QueryExecModeCacheStatement and automatically prepares and caches prepared statements. // However, this may be incompatible with proxies such as PGBouncer. diff --git a/internal/app/config/config.go b/internal/app/config/config.go index 000e69d..78e7638 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -5,10 +5,11 @@ import ( "fmt" "os" + "gopkg.in/yaml.v3" + "github.com/ozontech/seq-ui/internal/app/config/migrate" v1 "github.com/ozontech/seq-ui/internal/app/config/v1" v2 "github.com/ozontech/seq-ui/internal/app/config/v2" - "gopkg.in/yaml.v3" ) const ( From ebcf7a8c5489dcef27774809953d42e53467391d Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 29 Jul 2026 13:00:14 +0300 Subject: [PATCH 06/29] update --- cmd/seq-ui/main.go | 22 +++-- .../api/seqapi/v1/grpc/aggregation_test.go | 12 +-- internal/api/seqapi/v1/http/api.go | 54 ++++++++----- internal/api/seqapi/v1/http/events.go | 2 +- internal/api/seqapi/v1/http/fields.go | 26 ++---- internal/api/seqapi/v1/http/fields_test.go | 2 +- internal/api/seqapi/v1/http/logs_lifespan.go | 6 +- internal/api/seqapi/v1/http/search.go | 4 +- .../api/seqapi/v1/http/start_async_search.go | 9 ++- internal/app/auth/oidc.go | 17 +--- internal/app/config/v2/config.go | 48 +++++++++++ internal/app/mw/auth.go | 5 +- internal/app/server/init.go | 2 +- internal/app/server/server.go | 6 +- internal/pkg/cache/ctor.go | 80 +++++++++++++------ 15 files changed, 192 insertions(+), 103 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index c35fc71..f75116d 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -91,9 +91,20 @@ func run(ctx context.Context) { zap.Float64("sampler_param", tracingCfg.SamplerParam)) } - registrar := initApp(ctx, cfg) + logger.Info("initializing caches") + caches, err := cache.FromConfig(ctx, cfg.Cache) + if err != nil { + logger.Fatal("failed to init caches", zap.Error(err)) + } + + var oidcCache cache.Cache + if cfg.Server.Auth != nil && cfg.Server.Auth.OIDC != nil { + oidcCache = caches[cfg.Server.Auth.OIDC.CacheID] + } - serv, err := server.New(ctx, cfg.Server, registrar) + registrar := initApp(ctx, cfg, caches) + + serv, err := server.New(ctx, cfg.Server, registrar, oidcCache) if err != nil { logger.Fatal("app init error", zap.Error(err)) } @@ -105,7 +116,7 @@ func run(ctx context.Context) { } } -func initApp(ctx context.Context, cfg config.Config) *api.Registrar { +func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cache) *api.Registrar { logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { @@ -124,11 +135,6 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { massExportV1 = massexport_v1.New(exportServer) } - logger.Info("initializing caches") - caches, err := cache.FromConfig(ctx, cfg.Cache) - if err != nil { - logger.Fatal("failed to init caches", zap.Error(err)) - } logger.Info("initializing db") db, err := initDb(ctx, cfg.DB) diff --git a/internal/api/seqapi/v1/grpc/aggregation_test.go b/internal/api/seqapi/v1/grpc/aggregation_test.go index 785a824..7ee158b 100644 --- a/internal/api/seqapi/v1/grpc/aggregation_test.go +++ b/internal/api/seqapi/v1/grpc/aggregation_test.go @@ -12,7 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -50,7 +50,7 @@ func TestGetAggregation(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -65,7 +65,7 @@ func TestGetAggregation(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 2, }, }, @@ -82,7 +82,7 @@ func TestGetAggregation(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 1, }, }, @@ -178,7 +178,7 @@ func TestGetAggregationWithNormalization(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -221,7 +221,7 @@ func TestGetAggregationWithNormalization(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index bb567fc..b8c5f43 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -28,32 +28,43 @@ type apiParams struct { } type apiGlobalParams struct { - fieldsCache *fieldsCache - masker *mask.Masker - pinnedFields fields - systemFields fields + fieldsCache *fieldsCache + masker *mask.Masker + pinnedFields fields + systemFields fields + eventsCacheTTL time.Duration + logsLifespanCacheKey string + logsLifespanCacheTTL time.Duration + fieldsCacheTTL time.Duration } type API struct { - config config.SeqAPI - globalParams apiGlobalParams - paramsByEnv map[string]apiParams - cache cache.Cache - nowFn func() time.Time - asyncSearches asyncsearches.Service - envsResponse getEnvsResponse + config config.SeqAPI + globalParams apiGlobalParams + paramsByEnv map[string]apiParams + inmemWithRedisCache cache.Cache + redisCache cache.Cache + nowFn func() time.Time + asyncSearches asyncsearches.Service + envsResponse getEnvsResponse } func New( cfg config.SeqAPI, seqDBСlients map[string]seqdb.Client, - cache cache.Cache, + inmemWithRedisCache cache.Cache, + redisCache cache.Cache, asyncSearches asyncsearches.Service, ) *API { globalParams := apiGlobalParams{ - pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), - systemFields: parseFields(cfg.GlobalOptions.SystemFields), + pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), + systemFields: parseFields(cfg.GlobalOptions.SystemFields), + eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, + logsLifespanCacheKey: cfg.GlobalOptions.LogsLifespanCacheKey, + fieldsCacheTTL: cfg.GlobalOptions.FieldsCacheTTL, + logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, } + if cfg.GlobalOptions.FieldsCacheTTL > 0 { globalParams.fieldsCache = newFieldsCache(cfg.GlobalOptions.FieldsCacheTTL) } @@ -100,13 +111,14 @@ func New( } return &API{ - config: cfg, - globalParams: globalParams, - paramsByEnv: paramsByEnv, - cache: cache, - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + config: cfg, + globalParams: globalParams, + paramsByEnv: paramsByEnv, + inmemWithRedisCache: inmemWithRedisCache, + redisCache: redisCache, + nowFn: time.Now, + asyncSearches: asyncSearches, + envsResponse: parseEnvs(cfg), } } diff --git a/internal/api/seqapi/v1/http/events.go b/internal/api/seqapi/v1/http/events.go index 6f8154a..f56cd42 100644 --- a/internal/api/seqapi/v1/http/events.go +++ b/internal/api/seqapi/v1/http/events.go @@ -79,7 +79,7 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { } if data, err := proto.Marshal(resp.Event); err == nil { - _ = a.inmemWithRedisCache.SetWithTTL(ctx, id, string(data), params.options.EventsCacheTTL) + _ = a.inmemWithRedisCache.SetWithTTL(ctx, id, string(data), a.globalParams.eventsCacheTTL) } else { logger.Error("failed to marshal event proto for caching", zap.String("id", id), zap.Error(err)) } diff --git a/internal/api/seqapi/v1/http/fields.go b/internal/api/seqapi/v1/http/fields.go index 556e8f4..8e99fcc 100644 --- a/internal/api/seqapi/v1/http/fields.go +++ b/internal/api/seqapi/v1/http/fields.go @@ -34,7 +34,7 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - if params.fieldsCache == nil { + if a.globalParams.fieldsCache == nil { resp, err := params.client.GetFields(ctx, &seqapi.GetFieldsRequest{}) if err != nil { wr.Error(err, http.StatusInternalServerError) @@ -43,14 +43,14 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { res := getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, + SystemFields: a.globalParams.systemFields, + PinnedFields: a.globalParams.pinnedFields, } wr.WriteJson(res) return } - rawFields, cached, isActual := params.fieldsCache.getFields() + rawFields, cached, isActual := a.globalParams.fieldsCache.getFields() if cached && isActual { _, _ = wr.Write(rawFields) return @@ -70,8 +70,8 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { res := getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, + SystemFields: a.globalParams.systemFields, + PinnedFields: a.globalParams.pinnedFields, } resData, err := json.Marshal(res) if err != nil { @@ -85,7 +85,7 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - params.fieldsCache.setFields(resData) + a.globalParams.fieldsCache.setFields(resData) _, _ = wr.Write(resData) } @@ -98,18 +98,8 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { // @Success 200 {object} getFieldsResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" func (a *API) serveGetPinnedFields(w http.ResponseWriter, r *http.Request) { - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_pinned_fields") - defer span.End() - - env := getEnvFromContext(ctx) - params, err := a.GetEnvParams(env) - if err != nil { - httputil.NewWriter(w).Error(err, http.StatusBadRequest) - return - } - httputil.NewWriter(w).WriteJson(getFieldsResponse{ - Fields: params.pinnedFields, + Fields: a.globalParams.pinnedFields, }) } diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index 81bfb7b..3911f54 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -83,7 +83,7 @@ func TestServeGetFields(t *testing.T) { }, }, }, - cfg: config.SeqAPIOptions{ + cfg: config.Options{ SystemFields: []config.Field{ {Name: "field1", Type: "keyword"}, {Name: "field2", Type: "text"}, diff --git a/internal/api/seqapi/v1/http/logs_lifespan.go b/internal/api/seqapi/v1/http/logs_lifespan.go index 50f9a98..9e52e55 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan.go +++ b/internal/api/seqapi/v1/http/logs_lifespan.go @@ -37,9 +37,7 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { return } - cacheKey := params.options.LogsLifespanCacheKey - - if resStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { + if resStr, err := a.redisCache.Get(ctx, a.globalParams.logsLifespanCacheKey); err == nil { res := 0 res, err = strconv.Atoi(resStr) if err == nil { @@ -66,7 +64,7 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { res := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) - err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(res), params.options.LogsLifespanCacheTTL) + err = a.redisCache.SetWithTTL(ctx, a.globalParams.logsLifespanCacheKey, strconv.Itoa(res), a.globalParams.logsLifespanCacheTTL) if err != nil { logger.Error("can't set logs lifespan to cache", zap.Error(err)) } diff --git a/internal/api/seqapi/v1/http/search.go b/internal/api/seqapi/v1/http/search.go index fd19854..6916242 100644 --- a/internal/api/seqapi/v1/http/search.go +++ b/internal/api/seqapi/v1/http/search.go @@ -140,9 +140,9 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { } searchResp := searchResponseFromProto(resp, httpReq.WithTotal) - if params.masker != nil { + if a.globalParams.masker != nil { for i := range searchResp.Events { - params.masker.Mask(searchResp.Events[i].Data) + a.globalParams.masker.Mask(searchResp.Events[i].Data) } } diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index 1096ec8..f2b548c 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -37,6 +37,13 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_start_async_search") defer span.End() + env := getEnvFromContext(ctx) + params, err := a.GetEnvParams(env) + if err != nil { + httputil.NewWriter(w).Error(err, http.StatusBadRequest) + return + } + var httpReq startAsyncSearchRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) @@ -54,7 +61,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { continue } if err := api_error.CheckAggregationTsInterval(agg.Interval, httpReq.From, httpReq.To, - a.config.MaxBucketsPerAggregationTs, + params.options.MaxBucketsPerAggregationTs, ); err != nil { wr.Error(err, http.StatusBadRequest) return diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 15caf35..4a53756 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -13,7 +13,7 @@ import ( "github.com/coreos/go-oidc/v3/oidc" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/tls" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" @@ -42,21 +42,12 @@ type oidcProvider struct { allowedClients []string } -func NewOIDCProvider(ctx context.Context, cfg *config.OIDC, cacheCfg config.Cache) (OIDCProvider, error) { +func NewOIDCProvider(ctx context.Context, cfg *config.OIDC, oidcCache cache.Cache) (OIDCProvider, error) { var ( - err error - oidcCache cache.Cache - oidcCtx context.Context + err error + oidcCtx context.Context ) - if cfg.CacheSecretKey != "" { - logger.Info("initializing oidc cache") - oidcCache, err = cache.NewInmemoryWithRedisOrInmemory(ctx, cacheCfg) - if err != nil { - return nil, fmt.Errorf("init oidc cache: %w", err) - } - } - if !cfg.SkipVerify { oidcCtx, err = newHTTPContext( context.Background(), diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 9350474..d9c7110 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -277,6 +277,7 @@ type CORS struct { } type OIDC struct { + CacheID string `yaml:"cache_id"` SkipVerify bool `yaml:"skip_verify"` AuthURLs []string `yaml:"auth_urls"` RootCA string `yaml:"root_ca"` @@ -344,6 +345,34 @@ type Cache struct { Redis []Redis `yaml:"redis"` } +func (c *Cache) InmemByID(id string) *InmemoryCache { + if c == nil { + return nil + } + + for i := range c.Inmemory { + if c.Inmemory[i].ID == id { + return &c.Inmemory[i] + } + } + + return nil +} + +func (c *Cache) RedisByID(id string) *Redis { + if c == nil { + return nil + } + + for i := range c.Redis { + if c.Redis[i].ID == id { + return &c.Redis[i] + } + } + + return nil +} + type S3 struct { Endpoint string `yaml:"endpoint"` AccessKeyID string `yaml:"access_key_id"` @@ -472,6 +501,7 @@ type Field struct { type SeqAPI struct { CacheID string `yaml:"cache_id"` + RedisID string `yaml:"redis_id"` GlobalOptions SeqAPIGlobalOptions `yaml:"global_options"` Options SeqAPIOptions `yaml:"options"` Envs map[string]SeqAPIEnv `yaml:"envs"` @@ -687,6 +717,24 @@ func Normalize(cfg *Config) error { return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) } } + + if cfg.Server.Auth != nil && cfg.Server.Auth.OIDC != nil { + oidc := cfg.Server.Auth.OIDC + hasCacheKey := oidc.CacheSecretKey != "" + hasCacheID := oidc.CacheID != "" + + switch { + case hasCacheKey && !hasCacheID: + return fmt.Errorf("auth.oidc.cache_secret_key is set but auth.oidc.cache_id is empty") + case !hasCacheKey && hasCacheID: + return fmt.Errorf("auth.oidc.cache_id is set but auth.oidc.cache_secret_key is empty") + case hasCacheID && hasCacheKey: + if cfg.Cache.InmemByID(oidc.CacheID) == nil && cfg.Cache.RedisByID(oidc.CacheID) == nil { + return fmt.Errorf("auth.oidc.cache_id %q not found", oidc.CacheID) + } + } + } + return nil } diff --git a/internal/app/mw/auth.go b/internal/app/mw/auth.go index ef8ee74..cf7c2ad 100644 --- a/internal/app/mw/auth.go +++ b/internal/app/mw/auth.go @@ -8,6 +8,7 @@ import ( "github.com/ozontech/seq-ui/internal/app/auth" config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" ) @@ -31,7 +32,7 @@ type AuthProviders struct { func NewAuthProviders( ctx context.Context, jwtSecretKey string, oidcCfg *config.OIDC, - cacheCfg config.Cache, + oidcCache cache.Cache, ) (AuthProviders, error) { authPrvds := AuthProviders{} @@ -47,7 +48,7 @@ func NewAuthProviders( if oidcCfg != nil { logger.Info("initializing oidc provider") var err error - authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, oidcCfg, cacheCfg) + authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, oidcCfg, oidcCache) if err != nil { return authPrvds, fmt.Errorf("failed to init oidc provider: %w", err) } diff --git a/internal/app/server/init.go b/internal/app/server/init.go index affc9c8..b3545a8 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -30,7 +30,7 @@ var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { var err error - s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.Auth.JWT.SecretKey, s.config.Auth.OIDC, s.Cache) + s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.Auth.JWT.SecretKey, s.config.Auth.OIDC, s.oidcCache) if err != nil { return err } diff --git a/internal/app/server/server.go b/internal/app/server/server.go index 3d9e256..1b5a0ca 100644 --- a/internal/app/server/server.go +++ b/internal/app/server/server.go @@ -10,11 +10,13 @@ import ( "github.com/ozontech/seq-ui/internal/api" config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/mw" + "github.com/ozontech/seq-ui/internal/pkg/cache" ) // Server contains application dependencies. type Server struct { config *config.Server + oidcCache cache.Cache debugServer *http.Server grpcServer *grpc.Server httpServer *http.Server @@ -24,8 +26,8 @@ type Server struct { } // New returns a new Server. -func New(ctx context.Context, cfg *config.Server, registrar *api.Registrar) (*Server, error) { - s := &Server{config: cfg} +func New(ctx context.Context, cfg *config.Server, registrar *api.Registrar, oidcCache cache.Cache) (*Server, error) { + s := &Server{config: cfg, oidcCache: oidcCache} if err := s.init(ctx, registrar); err != nil { return nil, fmt.Errorf("init server: %w", err) diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index b9030c0..5cf2a1c 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -4,43 +4,77 @@ import ( "context" "fmt" - "go.uber.org/zap" - config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/logger" + "go.uber.org/zap" ) -func NewInmemoryWithRedisOrInmemory(ctx context.Context, cfg config.Cache) (Cache, error) { - return newRedisBasedOrInmemory(ctx, cfg, true) -} +func FromConfig(ctx context.Context, cfg *config.Cache) (map[string]Cache, error) { + caches := make(map[string]Cache, len(cfg.Redis)+len(cfg.Inmemory)) -func NewRedisOrInmemory(ctx context.Context, cfg config.Cache) (Cache, error) { - return newRedisBasedOrInmemory(ctx, cfg, false) -} + for i := range cfg.Redis { + redis := &cfg.Redis[i] + var inmem *config.InmemoryCache + if redis.WithInmemID != "" { + inmem = cfg.InmemByID(redis.WithInmemID) + } -func newRedisBasedOrInmemory(ctx context.Context, cfg config.Cache, withInmem bool) (Cache, error) { - inmem, err := newInmemoryCache(cfg.Inmemory) - if err != nil { - return nil, fmt.Errorf("init inmemory cache: %w", err) + cache, err := New(ctx, inmem, redis) + if err != nil { + return nil, fmt.Errorf("new cache %q: %w", redis.ID, err) + } + + caches[redis.ID] = cache } - if cfg.Redis == nil { - logger.Warn("redis cache config is nil; inmemory cache will be used instead") - return inmem, nil + for i := range cfg.Inmemory { + inmem := &cfg.Inmemory[i] + if hasWithInmem(inmem.ID, cfg.Redis) { + continue + } + + cache, err := newInmemoryCache(*inmem) + if err != nil { + return nil, fmt.Errorf("new inmemory cache %q: %w", inmem.ID, err) + } + + caches[inmem.ID] = cache } - redis, err := newRedisCache(ctx, cfg.Redis) + return caches, nil +} + +func New(ctx context.Context, inmemCfg *config.InmemoryCache, redisCfg *config.Redis) (Cache, error) { + redis, redisErr := newRedisCache(ctx, redisCfg) + if inmemCfg == nil { + if redisErr != nil { + return nil, fmt.Errorf("init redis cache: %w", redisErr) + } + return redis, nil + } + + inmem, err := newInmemoryCache(*inmemCfg) if err != nil { + return nil, fmt.Errorf("init inmemory cache: %w", err) + } + + if redisErr != nil { logger.Warn("failed to init redis cache; inmemory cache will be used instead", zap.Error(err)) - return inmem, nil + return inmem, err } - if withInmem { - return &inmemWithRedis{ - inmem: inmem, - redis: redis, - }, nil + return &inmemWithRedis{ + inmem: inmem, + redis: redis, + }, nil +} + +func hasWithInmem(inmemID string, redisCfgs []config.Redis) bool { + for _, r := range redisCfgs { + if r.WithInmemID == inmemID { + return true + } } - return redis, nil + return false } From 7f3a924b353cd5c85c8443ed8c7ad5b8c3ecbcf6 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 29 Jul 2026 14:48:12 +0300 Subject: [PATCH 07/29] refactor --- cmd/seq-ui/main.go | 29 +++++++++++++++++++++-------- internal/app/config/v2/config.go | 14 ++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index f75116d..4eb39bb 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -168,15 +168,15 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesService) - logger.Info("initializing clickhouse") - ch, err := initClickHouse(ctx, cfg.Server.CH) + logger.Info("initializing clickhouse clients") + chClients, err := initClickHouseClients(ctx, cfg) if err != nil { - logger.Fatal("failed to init clickhouse", zap.Error(err)) + logger.Fatal("failed to init clickhouse clients", zap.Error(err)) } var errorGroupsV1 *errorgroups_v1.ErrorGroups - if ch != nil { - repo := repositorych.New(ch, cfg.Clients.ClickHouse.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) + if chClients != nil { + repo := repositorych.New(chClients[cfg.Handlers.ErrorGroups.CHID], cfg.Clients.ClickHouse.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) svc := errorgroups.New(repo, cfg.Handlers.ErrorGroups.LogTagsMapping) errorGroupsV1 = errorgroups_v1.New(svc) @@ -186,7 +186,7 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac } func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb.Client, error) { - clients := make(map[string]seqdb.Client) + clients := make(map[string]seqdb.Client, len(cfg.Clients.SeqDB)) for _, clientCfg := range cfg.Clients.SeqDB { client, err := createSeqBDClient(ctx, clientCfg, cfg.Handlers.SeqAPI) if err != nil { @@ -230,7 +230,7 @@ func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI confi return seqdb.NewGRPCClient(ctx, clientParams) } -func initDb(ctx context.Context, cfg *config_v2.DB) (*pgxpool.Pool, error) { +func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { if cfg == nil { logger.Warn("db config is nil, running without db") return nil, nil @@ -270,7 +270,20 @@ func initExportService(ctx context.Context, cfg config.MassExport, client seqdb. return massexport.NewService(ctx, cfg, sessionStore, fileStore, client) } -func initClickHouse(ctx context.Context, cfg *config.CH) (driver.Conn, error) { +func initClickHouseClients(ctx context.Context, cfg config.Config) (map[string]driver.Conn, error) { + clients := make(map[string]driver.Conn, len(cfg.Clients.ClickHouse)) + for _, clientCfg := range cfg.Clients.ClickHouse { + client, err := createClickHouseClient(ctx, &clientCfg) + if err != nil { + return nil, fmt.Errorf("failed to create clickhouse client %s: %w", clientCfg.ID, err) + } + clients[clientCfg.ID] = client + } + + return clients, nil +} + +func createClickHouseClient(ctx context.Context, cfg *config.CHClient) (driver.Conn, error) { if cfg == nil { logger.Warn("clickhouse config is nil, running without clickhouse") return nil, nil diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index d9c7110..286e836 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -487,6 +487,20 @@ type Clients struct { ClickHouse []CHClient `yaml:"clickhouse"` } +func (c *Clients) ClickHouseByID(id string) *CHClient { + if c == nil { + return nil + } + + for i := range c.ClickHouse { + if c.ClickHouse[i].ID == id { + return &c.ClickHouse[i] + } + } + + return nil +} + type Handlers struct { SeqAPI SeqAPI `yaml:"seq_api"` ErrorGroups ErrorGroups `yaml:"error_groups"` From cb6d68cc76ee8fd64a932b6c384a0edf3acb169b Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Thu, 30 Jul 2026 02:35:38 +0300 Subject: [PATCH 08/29] big update --- cmd/seq-ui/main.go | 62 +++++------ internal/api/seqapi/v1/grpc/aggregation.go | 4 +- internal/api/seqapi/v1/grpc/api.go | 103 ++++++++---------- internal/api/seqapi/v1/grpc/events.go | 10 +- internal/api/seqapi/v1/grpc/events_test.go | 6 +- internal/api/seqapi/v1/grpc/fields.go | 33 ++---- internal/api/seqapi/v1/grpc/fields_test.go | 12 +- internal/api/seqapi/v1/grpc/get_envs_test.go | 8 +- internal/api/seqapi/v1/grpc/limits_test.go | 4 +- internal/api/seqapi/v1/grpc/logs_lifespan.go | 6 +- .../api/seqapi/v1/grpc/logs_lifespan_test.go | 4 +- internal/api/seqapi/v1/grpc/search.go | 4 +- internal/api/seqapi/v1/grpc/search_test.go | 14 +-- internal/api/seqapi/v1/grpc/test_data.go | 10 +- internal/api/seqapi/v1/http/api.go | 8 +- internal/api/seqapi/v1/http/events_test.go | 6 +- internal/api/seqapi/v1/http/export_test.go | 14 +-- internal/api/seqapi/v1/http/fields_test.go | 12 +- internal/api/seqapi/v1/http/get_envs_test.go | 15 ++- internal/api/seqapi/v1/http/limits_test.go | 4 +- .../api/seqapi/v1/http/logs_lifespan_test.go | 4 +- internal/api/seqapi/v1/http/search_test.go | 26 ++--- internal/api/seqapi/v1/http/test_data.go | 10 +- internal/api/seqapi/v1/seqapi.go | 2 +- internal/app/config/migrate/v1_v2.go | 10 +- internal/app/config/v2/config.go | 93 ++++++++-------- internal/pkg/cache/ctor.go | 3 +- .../seqdb/seqproxyapi/v1/seq_proxy_api.pb.go | 2 +- .../seqproxyapi/v1/seq_proxy_api_grpc.pb.go | 2 +- .../massexport/seq_proxy_downloader.go | 4 +- internal/pkg/service/massexport/service.go | 2 +- .../service/massexport/sessionstore/redis.go | 16 +-- pkg/dashboards/v1/dashboards.pb.go | 2 +- pkg/dashboards/v1/dashboards_grpc.pb.go | 2 +- pkg/errorgroups/v1/errorgroups.pb.go | 2 +- pkg/errorgroups/v1/errorgroups_grpc.pb.go | 2 +- pkg/massexport/v1/massexport.pb.go | 2 +- pkg/massexport/v1/massexport_grpc.pb.go | 2 +- pkg/seqapi/v1/seq_api.pb.go | 2 +- pkg/seqapi/v1/seq_api_grpc.pb.go | 2 +- pkg/userprofile/v1/userprofile.pb.go | 2 +- pkg/userprofile/v1/userprofile_grpc.pb.go | 2 +- 42 files changed, 250 insertions(+), 283 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 4eb39bb..fdadaeb 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -120,20 +120,13 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { - logger.Fatal("failed to init seq-db client", zap.Error(err)) + logger.Fatal("failed to init seq-db clients", zap.Error(err)) } - var massExportV1 *massexport_v1.MassExport - if cfg.Handlers.MassExport != nil { - massExportClient, ok := seqDBClients[cfg.Handlers.MassExport.SeqDBID] - if !ok { - logger.Fatal("mass_export seq_db client not found", zap.String("seq_db_id", cfg.Handlers.MassExport.SeqDBID)) - } - exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, massExportClient) - if err != nil { - logger.Fatal("can't init export server", zap.Error(err)) - } - massExportV1 = massexport_v1.New(exportServer) + logger.Info("initializing clickhouse clients") + chClients, err := initClickHouseClients(ctx, cfg) + if err != nil { + logger.Fatal("failed to init clickhouse clients", zap.Error(err)) } logger.Info("initializing db") @@ -142,6 +135,18 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac logger.Fatal("failed to init db", zap.Error(err)) } + var massExportV1 *massexport_v1.MassExport + if cfg.Handlers.MassExport != nil { + me := cfg.Handlers.MassExport + redisCfg := cfg.Cache.RedisByID(me.SessionStore.RedisID) + + exportServer, err := initExportService(ctx, *me, redisCfg, seqDBClients[cfg.Handlers.MassExport.SeqDBID]) + if err != nil { + logger.Fatal("can't init export server", zap.Error(err)) + } + massExportV1 = massexport_v1.New(exportServer) + } + var ( asyncSearchesService asyncsearches.Service userProfileV1 *userprofile_v1.UserProfile @@ -156,27 +161,17 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac userProfileV1 = userprofile_v1.New(userProfilesSvc) dashboardsV1 = dashboards_v1.New(dashboardsSvc) - asyncSearchClient, ok := seqDBClients[cfg.Handlers.AsyncSearch.SeqDBID] - if !ok { - logger.Fatal("async_search seq_db client not found", zap.String("seq_db_id", cfg.Handlers.AsyncSearch.SeqDBID)) - } - asyncSearchesService = asyncsearches.New(ctx, repo, asyncSearchClient, cfg.Handlers.AsyncSearch) + asyncSearchesService = asyncsearches.New(ctx, repo, seqDBClients[cfg.Handlers.AsyncSearch.SeqDBID], cfg.Handlers.AsyncSearch) } seqApiCache := caches[cfg.Handlers.SeqAPI.CacheID] seqApiRedisCache := caches[cfg.Handlers.SeqAPI.RedisID] - - seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesService) - - logger.Info("initializing clickhouse clients") - chClients, err := initClickHouseClients(ctx, cfg) - if err != nil { - logger.Fatal("failed to init clickhouse clients", zap.Error(err)) - } + seqApiV1 := seqapi_v1.New(&cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesService) var errorGroupsV1 *errorgroups_v1.ErrorGroups - if chClients != nil { - repo := repositorych.New(chClients[cfg.Handlers.ErrorGroups.CHID], cfg.Clients.ClickHouse.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) + if cfg.Handlers.ErrorGroups != nil { + chCfg := cfg.Clients.ClickHouseByID(cfg.Handlers.ErrorGroups.CHID) + repo := repositorych.New(chClients[cfg.Handlers.ErrorGroups.CHID], chCfg.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) svc := errorgroups.New(repo, cfg.Handlers.ErrorGroups.LogTagsMapping) errorGroupsV1 = errorgroups_v1.New(svc) @@ -188,7 +183,7 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb.Client, error) { clients := make(map[string]seqdb.Client, len(cfg.Clients.SeqDB)) for _, clientCfg := range cfg.Clients.SeqDB { - client, err := createSeqBDClient(ctx, clientCfg, cfg.Handlers.SeqAPI) + client, err := createSeqBDClient(ctx, &clientCfg, &cfg.Handlers.SeqAPI) if err != nil { return nil, fmt.Errorf("failed to create seq_db client %s: %w", clientCfg.ID, err) } @@ -198,7 +193,7 @@ func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb. return clients, nil } -func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI config.SeqAPI) (seqdb.Client, error) { +func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *config.SeqAPI) (seqdb.Client, error) { maxSearchLimit := int(seqAPI.Options.MaxSearchLimit) for _, env := range seqAPI.Envs { if env.SeqDBID != cfg.ID { @@ -254,8 +249,8 @@ func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { return pool, nil } -func initExportService(ctx context.Context, cfg config.MassExport, client seqdb.Client) (massexport.Service, error) { - sessionStore, err := sessionstore.NewRedisSessionStore(ctx, cfg.SessionStore) +func initExportService(ctx context.Context, cfg config.MassExport, redisCfg *config.Redis, client seqdb.Client) (massexport.Service, error) { + sessionStore, err := sessionstore.NewRedisSessionStore(ctx, redisCfg, cfg.SessionStore.ExportLifetime) if err != nil { return nil, fmt.Errorf("init session store: %w", err) } @@ -284,11 +279,6 @@ func initClickHouseClients(ctx context.Context, cfg config.Config) (map[string]d } func createClickHouseClient(ctx context.Context, cfg *config.CHClient) (driver.Conn, error) { - if cfg == nil { - logger.Warn("clickhouse config is nil, running without clickhouse") - return nil, nil - } - conn, err := clickhouse.Open(&clickhouse.Options{ Addr: cfg.Addrs, Auth: clickhouse.Auth{ diff --git a/internal/api/seqapi/v1/grpc/aggregation.go b/internal/api/seqapi/v1/grpc/aggregation.go index e8fd760..9603b88 100644 --- a/internal/api/seqapi/v1/grpc/aggregation.go +++ b/internal/api/seqapi/v1/grpc/aggregation.go @@ -80,7 +80,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ return nil, err } - if params.masker != nil { + if a.globalParams.masker != nil { buf := make([]string, 0) for i, agg := range resp.Aggregations { if agg == nil { @@ -98,7 +98,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ field = aggReq.GroupBy } - buf = params.masker.MaskAgg(field, buf) + buf = a.globalParams.masker.MaskAgg(field, buf) for j, key := range buf { if agg.Buckets[j] != nil { diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index da9a784..cc75ad0 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -10,7 +10,7 @@ import ( "go.uber.org/zap" "google.golang.org/grpc/metadata" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" @@ -21,19 +21,25 @@ import ( ) type apiParams struct { - client seqdb.Client - options *config.SeqAPIOptions - fieldsCache *fieldsCache - masker *mask.Masker - pinnedFields []*seqapi.Field - systemFields []*seqapi.Field + client seqdb.Client + options *config.SeqAPIOptions +} + +type apiGlobalParams struct { + fieldsCache *fieldsCache + masker *mask.Masker + pinnedFields []*seqapi.Field + systemFields []*seqapi.Field + eventsCacheTTL time.Duration + logsLifespanCacheKey string + logsLifespanCacheTTL time.Duration } type API struct { seqapi.UnimplementedSeqAPIServiceServer - config config.SeqAPI - params apiParams + config *config.SeqAPI + globalParams apiGlobalParams paramsByEnv map[string]apiParams inmemWithRedisCache cache.Cache redisCache cache.Cache @@ -43,76 +49,57 @@ type API struct { } func New( - cfg config.SeqAPI, + cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, asyncSearches asyncsearches.Service, ) *API { - var globalfCache *fieldsCache - if cfg.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.FieldsCacheTTL) + globalParams := apiGlobalParams{ + pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), + systemFields: parseFields(cfg.GlobalOptions.SystemFields), + eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, + logsLifespanCacheKey: cfg.GlobalOptions.LogsLifespanCacheKey, + logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, + } + + if cfg.GlobalOptions.FieldsCacheTTL > 0 { + globalParams.fieldsCache = newFieldsCache(cfg.GlobalOptions.FieldsCacheTTL) } - globalMasker, err := mask.New(cfg.Masking) + globalMasker, err := mask.New(cfg.GlobalOptions.Masking) if err != nil { logger.Fatal("failed to init masking", zap.Error(err)) } + globalParams.masker = globalMasker - globalPinnedFields := parseFields(cfg.PinnedFields) - globalSystemFields := parseFields(cfg.SystemFields) - - var params apiParams - var paramsByEnv map[string]apiParams - + paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { - paramsByEnv = make(map[string]apiParams) for envName, envConfig := range cfg.Envs { - client := seqDBСlients[envConfig.SeqDB] - options := envConfig.Options - - var envfCache *fieldsCache - if options.FieldsCacheTTL > 0 { - envfCache = newFieldsCache(options.FieldsCacheTTL) - } - - var envMasker, err = mask.New(options.Masking) - if err != nil { - logger.Fatal("failed to init env masking", zap.Error(err)) - } - - envPinnedFields := parseFields(options.PinnedFields) - envSystemFields := parseFields(options.SystemFields) - paramsByEnv[envName] = apiParams{ - client: client, - options: options, - fieldsCache: envfCache, - masker: envMasker, - pinnedFields: envPinnedFields, - systemFields: envSystemFields, + client: seqDBСlients[envConfig.SeqDBID], + options: envConfig.Options, } } } else { - client, exists := seqDBСlients[config.DefaultSeqDBClientID] - if !exists { - logger.Fatal("default client not found", - zap.String("clientID", config.DefaultSeqDBClientID)) + if len(seqDBСlients) != 1 { + logger.Fatal("seq_api.envs is empty, configure envs when clients.seq_db has more than one instance") + } + + var client seqdb.Client + for _, c := range seqDBСlients { + client = c } - params = apiParams{ - client: client, - options: cfg.SeqAPIOptions, - fieldsCache: globalfCache, - masker: globalMasker, - pinnedFields: globalPinnedFields, - systemFields: globalSystemFields, + paramsByEnv[""] = apiParams{ + client: client, + options: &cfg.Options, } } return &API{ config: cfg, - params: params, + globalParams: globalParams, paramsByEnv: paramsByEnv, inmemWithRedisCache: inmemWithRedisCache, redisCache: redisCache, @@ -133,7 +120,7 @@ func parseFields(fields []config.Field) []*seqapi.Field { return res } -func parseEnvs(cfg config.SeqAPI) *seqapi.GetEnvsResponse { +func parseEnvs(cfg *config.SeqAPI) *seqapi.GetEnvsResponse { var envs []*seqapi.GetEnvsResponse_Env if len(cfg.Envs) > 0 { // sort environment names to ensure deterministic output @@ -177,7 +164,7 @@ func parseEnvs(cfg config.SeqAPI) *seqapi.GetEnvsResponse { envs = append(envs, createEnvInfo(name, envConfig.Options)) } } else { - envs = []*seqapi.GetEnvsResponse_Env{createEnvInfo("", cfg.SeqAPIOptions)} + envs = []*seqapi.GetEnvsResponse_Env{createEnvInfo("", &cfg.Options)} } return &seqapi.GetEnvsResponse{Envs: envs} } @@ -193,7 +180,7 @@ func (a *API) GetEnvFromContext(ctx context.Context) string { func (a *API) GetParams(env string) (apiParams, error) { if len(a.config.Envs) == 0 { - return a.params, nil + return a.paramsByEnv[""], nil } if env == "" { diff --git a/internal/api/seqapi/v1/grpc/events.go b/internal/api/seqapi/v1/grpc/events.go index a4665b9..aea707b 100644 --- a/internal/api/seqapi/v1/grpc/events.go +++ b/internal/api/seqapi/v1/grpc/events.go @@ -41,8 +41,8 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap if cached, err := a.inmemWithRedisCache.Get(ctx, req.Id); err == nil { event := &seqapi.Event{} if err = proto.Unmarshal([]byte(cached), event); err == nil { - if params.masker != nil { - params.masker.Mask(event.Data) + if a.globalParams.masker != nil { + a.globalParams.masker.Mask(event.Data) } return &seqapi.GetEventResponse{Event: event}, nil } @@ -53,12 +53,12 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap return nil, err } - if params.masker != nil && resp.Event != nil { - params.masker.Mask(resp.Event.Data) + if a.globalParams.masker != nil && resp.Event != nil { + a.globalParams.masker.Mask(resp.Event.Data) } if data, err := proto.Marshal(resp.Event); err == nil { - _ = a.inmemWithRedisCache.SetWithTTL(ctx, req.Id, string(data), params.options.EventsCacheTTL) + _ = a.inmemWithRedisCache.SetWithTTL(ctx, req.Id, string(data), a.globalParams.eventsCacheTTL) } else { logger.Error("failed to marshal event proto for caching", zap.String("id", req.Id), zap.Error(err)) } diff --git a/internal/api/seqapi/v1/grpc/events_test.go b/internal/api/seqapi/v1/grpc/events_test.go index 3f10c1f..b910470 100644 --- a/internal/api/seqapi/v1/grpc/events_test.go +++ b/internal/api/seqapi/v1/grpc/events_test.go @@ -14,7 +14,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -104,7 +104,7 @@ func TestGetEvent(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ EventsCacheTTL: cacheTTL, }, }, @@ -369,7 +369,7 @@ func TestGetEventWithMasking(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ EventsCacheTTL: cacheTTL, Masking: tt.maskingCfg, }, diff --git a/internal/api/seqapi/v1/grpc/fields.go b/internal/api/seqapi/v1/grpc/fields.go index bae2836..f07109f 100644 --- a/internal/api/seqapi/v1/grpc/fields.go +++ b/internal/api/seqapi/v1/grpc/fields.go @@ -22,23 +22,23 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq return nil, status.Error(codes.InvalidArgument, err.Error()) } - if params.fieldsCache == nil { + if a.globalParams.fieldsCache == nil { resp, err := params.client.GetFields(ctx, req) if err != nil { return nil, err } - resp.SystemFields = params.systemFields - resp.PinnedFields = params.pinnedFields + resp.SystemFields = a.globalParams.systemFields + resp.PinnedFields = a.globalParams.pinnedFields return resp, nil } - fields, cached, isActual := params.fieldsCache.getFields() + fields, cached, isActual := a.globalParams.fieldsCache.getFields() if cached && isActual { return &seqapi.GetFieldsResponse{ Fields: fields, - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, + SystemFields: a.globalParams.systemFields, + PinnedFields: a.globalParams.pinnedFields, }, nil } @@ -48,31 +48,22 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq logger.Error("can't get fields; use cached fields", zap.Error(err)) return &seqapi.GetFieldsResponse{ Fields: fields, - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, + SystemFields: a.globalParams.systemFields, + PinnedFields: a.globalParams.pinnedFields, }, nil } return nil, err } - params.fieldsCache.setFields(resp.GetFields()) - resp.SystemFields = params.systemFields - resp.PinnedFields = params.pinnedFields + a.globalParams.fieldsCache.setFields(resp.GetFields()) + resp.SystemFields = a.globalParams.systemFields + resp.PinnedFields = a.globalParams.pinnedFields return resp, nil } func (a *API) GetPinnedFields(ctx context.Context, _ *seqapi.GetFieldsRequest) (*seqapi.GetFieldsResponse, error) { - ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_fields") - defer span.End() - - env := a.GetEnvFromContext(ctx) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - return &seqapi.GetFieldsResponse{ - Fields: params.pinnedFields, + Fields: a.globalParams.pinnedFields, }, nil } diff --git a/internal/api/seqapi/v1/grpc/fields_test.go b/internal/api/seqapi/v1/grpc/fields_test.go index f52149d..5df65b0 100644 --- a/internal/api/seqapi/v1/grpc/fields_test.go +++ b/internal/api/seqapi/v1/grpc/fields_test.go @@ -11,7 +11,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -20,7 +20,7 @@ func TestGetFields(t *testing.T) { tests := []struct { name string - cfg config.SeqAPIOptions + cfg config.SeqAPIGlobalOptions seqDBResp *seqapi.GetFieldsResponse wantResp *seqapi.GetFieldsResponse @@ -55,7 +55,7 @@ func TestGetFields(t *testing.T) { }, { name: "ok_with_system_and_pinned_fields", - cfg: config.SeqAPIOptions{ + cfg: config.SeqAPIGlobalOptions{ SystemFields: []config.Field{ {Name: "field1", Type: "keyword"}, {Name: "field2", Type: "text"}, @@ -121,7 +121,7 @@ func TestGetFields(t *testing.T) { SeqDB: seqDbMock, }, Cfg: config.SeqAPI{ - SeqAPIOptions: &tt.cfg, + GlobalOptions: tt.cfg, }, } @@ -173,7 +173,7 @@ func TestGetFieldsCached(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ FieldsCacheTTL: ttl, }, }, @@ -222,7 +222,7 @@ func TestGetPinnedFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ PinnedFields: tt.fields, }, }, diff --git a/internal/api/seqapi/v1/grpc/get_envs_test.go b/internal/api/seqapi/v1/grpc/get_envs_test.go index 5f5a1a4..4deb8be 100644 --- a/internal/api/seqapi/v1/grpc/get_envs_test.go +++ b/internal/api/seqapi/v1/grpc/get_envs_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -20,7 +20,7 @@ func TestGetEnvs(t *testing.T) { { name: "single_env", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 100, MaxExportLimit: 200, MaxParallelExportRequests: 2, @@ -144,8 +144,8 @@ func TestGetEnvs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() api := API{ - config: tt.cfg, - envsResponse: parseEnvs(tt.cfg), + config: &tt.cfg, + envsResponse: parseEnvs(&tt.cfg), } resp, err := api.GetEnvs(context.TODO(), &seqapi.GetEnvsRequest{}) require.NoError(t, err) diff --git a/internal/api/seqapi/v1/grpc/limits_test.go b/internal/api/seqapi/v1/grpc/limits_test.go index 30892c6..f38efe3 100644 --- a/internal/api/seqapi/v1/grpc/limits_test.go +++ b/internal/api/seqapi/v1/grpc/limits_test.go @@ -8,7 +8,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -21,7 +21,7 @@ func TestGetLimits(t *testing.T) { { name: "ok", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 100, MaxExportLimit: 200, MaxParallelExportRequests: 2, diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan.go b/internal/api/seqapi/v1/grpc/logs_lifespan.go index d54eb89..09ed1fc 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan.go @@ -30,9 +30,7 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ return nil, status.Error(codes.InvalidArgument, err.Error()) } - cacheKey := params.options.LogsLifespanCacheKey - - if countStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { + if countStr, err := a.redisCache.Get(ctx, a.globalParams.logsLifespanCacheKey); err == nil { count := 0 count, err = strconv.Atoi(countStr) if err == nil { @@ -58,7 +56,7 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ count := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) res := time.Duration(count) * lifespan.MeasureUnit - err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(count), params.options.LogsLifespanCacheTTL) + err = a.redisCache.SetWithTTL(ctx, a.globalParams.logsLifespanCacheKey, strconv.Itoa(count), a.globalParams.logsLifespanCacheTTL) if err != nil { logger.Error("can't set logs lifespan to cache", zap.Error(err)) } diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index 9b2fc2f..a82cb88 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -13,7 +13,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" @@ -106,7 +106,7 @@ func TestGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ LogsLifespanCacheKey: cacheKey, LogsLifespanCacheTTL: cacheTTL, }, diff --git a/internal/api/seqapi/v1/grpc/search.go b/internal/api/seqapi/v1/grpc/search.go index 7fb45fb..02149cf 100644 --- a/internal/api/seqapi/v1/grpc/search.go +++ b/internal/api/seqapi/v1/grpc/search.go @@ -123,9 +123,9 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se } } - if params.masker != nil { + if a.globalParams.masker != nil { for _, e := range resp.Events { - params.masker.Mask(e.Data) + a.globalParams.masker.Mask(e.Data) } } diff --git a/internal/api/seqapi/v1/grpc/search_test.go b/internal/api/seqapi/v1/grpc/search_test.go index b818531..ce1644c 100644 --- a/internal/api/seqapi/v1/grpc/search_test.go +++ b/internal/api/seqapi/v1/grpc/search_test.go @@ -12,7 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -62,7 +62,7 @@ func TestSearch(t *testing.T) { }, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxAggregationsPerRequest: 5, }, @@ -81,7 +81,7 @@ func TestSearch(t *testing.T) { Limit: 10, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }, @@ -98,7 +98,7 @@ func TestSearch(t *testing.T) { }, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxAggregationsPerRequest: 2, }, @@ -115,7 +115,7 @@ func TestSearch(t *testing.T) { Offset: 11, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxSearchOffsetLimit: 10, }, @@ -145,7 +145,7 @@ func TestSearch(t *testing.T) { }, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxSearchTotalLimit: int64(limit), }, @@ -161,7 +161,7 @@ func TestSearch(t *testing.T) { Offset: 0, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index a088897..ce8ed4e 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -5,7 +5,7 @@ import ( "time" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) @@ -18,15 +18,11 @@ var ( ) func setupTestAPI(data test.APITestData) *API { - // when test cases don't explicitly provide configuration - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} - } seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB for _, envConfig := range data.Cfg.Envs { - seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB + seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB } var asyncSvc asyncsearches.Service @@ -34,5 +30,5 @@ func setupTestAPI(data test.APITestData) *API { asyncSvc = data.Mocks.AsyncSearchesSvc } - return New(data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) + return New(&data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) } diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index b8c5f43..2f5b4b0 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -35,11 +35,10 @@ type apiGlobalParams struct { eventsCacheTTL time.Duration logsLifespanCacheKey string logsLifespanCacheTTL time.Duration - fieldsCacheTTL time.Duration } type API struct { - config config.SeqAPI + config *config.SeqAPI globalParams apiGlobalParams paramsByEnv map[string]apiParams inmemWithRedisCache cache.Cache @@ -50,7 +49,7 @@ type API struct { } func New( - cfg config.SeqAPI, + cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, @@ -61,7 +60,6 @@ func New( systemFields: parseFields(cfg.GlobalOptions.SystemFields), eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, logsLifespanCacheKey: cfg.GlobalOptions.LogsLifespanCacheKey, - fieldsCacheTTL: cfg.GlobalOptions.FieldsCacheTTL, logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, } @@ -161,7 +159,7 @@ func parseFields(fields []config.Field) []field { return res } -func parseEnvs(cfg config.SeqAPI) getEnvsResponse { +func parseEnvs(cfg *config.SeqAPI) getEnvsResponse { var envs []envInfo if len(cfg.Envs) > 0 { // sort environment names to ensure deterministic output diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index 7b33966..619fe6d 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -14,7 +14,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -121,7 +121,7 @@ func TestServeGetEvent(t *testing.T) { ctrl := gomock.NewController(t) seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ EventsCacheTTL: cacheTTL, }, }, @@ -381,7 +381,7 @@ func TestGetEventWithMasking(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ EventsCacheTTL: cacheTTL, Masking: tt.maskingCfg, }, diff --git a/internal/api/seqapi/v1/http/export_test.go b/internal/api/seqapi/v1/http/export_test.go index d203aa3..5c36e14 100644 --- a/internal/api/seqapi/v1/http/export_test.go +++ b/internal/api/seqapi/v1/http/export_test.go @@ -10,7 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -40,7 +40,7 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 100, MaxParallelExportRequests: 1, }, @@ -67,7 +67,7 @@ func TestServeExport(t *testing.T) { Fields: []string{"field1", "field2"}, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 100, MaxParallelExportRequests: 1, }, @@ -94,7 +94,7 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxParallelExportRequests: 0, }, }, @@ -110,7 +110,7 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 5, MaxParallelExportRequests: 1, }, @@ -128,7 +128,7 @@ func TestServeExport(t *testing.T) { Format: efCSV, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 100, MaxParallelExportRequests: 1, }, @@ -145,7 +145,7 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 100, MaxParallelExportRequests: 1, }, diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index 3911f54..3d6a26b 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -9,7 +9,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -23,7 +23,7 @@ func TestServeGetFields(t *testing.T) { tests := []struct { name string - cfg config.SeqAPIOptions + cfg config.SeqAPIGlobalOptions want getFieldsResponse wantErr bool @@ -83,7 +83,7 @@ func TestServeGetFields(t *testing.T) { }, }, }, - cfg: config.Options{ + cfg: config.SeqAPIGlobalOptions{ SystemFields: []config.Field{ {Name: "field1", Type: "keyword"}, {Name: "field2", Type: "text"}, @@ -110,7 +110,7 @@ func TestServeGetFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &tt.cfg, + GlobalOptions: tt.cfg, }, } @@ -198,7 +198,7 @@ func TestServeGetFieldsCached(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ FieldsCacheTTL: ttl, }, }, @@ -256,7 +256,7 @@ func TestServeGetPinnedFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ PinnedFields: tt.fields, }, }, diff --git a/internal/api/seqapi/v1/http/get_envs_test.go b/internal/api/seqapi/v1/http/get_envs_test.go index 3ede5d5..53ddae3 100644 --- a/internal/api/seqapi/v1/http/get_envs_test.go +++ b/internal/api/seqapi/v1/http/get_envs_test.go @@ -6,7 +6,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestServeGetEnvs(t *testing.T) { @@ -18,7 +18,7 @@ func TestServeGetEnvs(t *testing.T) { { name: "single_env", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 100, MaxExportLimit: 200, MaxParallelExportRequests: 2, @@ -42,10 +42,9 @@ func TestServeGetEnvs(t *testing.T) { { name: "ok_multiple_envs", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{}, Envs: map[string]config.SeqAPIEnv{ "cluster-220": { - SeqDB: "pro-seqdb", + SeqDBID: "pro-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 1000, MaxExportLimit: 500, @@ -55,7 +54,7 @@ func TestServeGetEnvs(t *testing.T) { }, }, "cluster-10": { - SeqDB: "prod-seqdb", + SeqDBID: "prod-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 1000, MaxExportLimit: 500, @@ -65,7 +64,7 @@ func TestServeGetEnvs(t *testing.T) { }, }, "cluster-102": { - SeqDB: "staging-seqdb", + SeqDBID: "staging-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 500, MaxExportLimit: 250, @@ -75,7 +74,7 @@ func TestServeGetEnvs(t *testing.T) { }, }, "prod": { - SeqDB: "stag-seqdb", + SeqDBID: "stag-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 500, MaxExportLimit: 250, @@ -85,7 +84,7 @@ func TestServeGetEnvs(t *testing.T) { }, }, "wyanki": { - SeqDB: "sta-seqdb", + SeqDBID: "sta-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 500, MaxExportLimit: 250, diff --git a/internal/api/seqapi/v1/http/limits_test.go b/internal/api/seqapi/v1/http/limits_test.go index cf339e4..1a8dca9 100644 --- a/internal/api/seqapi/v1/http/limits_test.go +++ b/internal/api/seqapi/v1/http/limits_test.go @@ -6,7 +6,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestServeGetLimits(t *testing.T) { @@ -21,7 +21,7 @@ func TestServeGetLimits(t *testing.T) { name: "ok", env: "default", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 100, MaxExportLimit: 200, MaxParallelExportRequests: 2, diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index 0ed4b5c..692ef86 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -12,7 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" @@ -103,7 +103,7 @@ func TestServeGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ LogsLifespanCacheKey: cacheKey, LogsLifespanCacheTTL: cacheTTL, }, diff --git a/internal/api/seqapi/v1/http/search_test.go b/internal/api/seqapi/v1/http/search_test.go index c22272f..9e92656 100644 --- a/internal/api/seqapi/v1/http/search_test.go +++ b/internal/api/seqapi/v1/http/search_test.go @@ -11,7 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -51,7 +51,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -87,7 +87,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -124,7 +124,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -163,7 +163,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -211,7 +211,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -255,7 +255,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -289,7 +289,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: true, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -330,7 +330,7 @@ func TestServeSearch(t *testing.T) { Limit: 10, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }, @@ -346,7 +346,7 @@ func TestServeSearch(t *testing.T) { Aggregations: aggregationQueries{{}, {}, {}}, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxAggregationsPerRequest: 2, }, @@ -363,7 +363,7 @@ func TestServeSearch(t *testing.T) { Offset: 11, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxSearchOffsetLimit: 10, }, @@ -386,7 +386,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxSearchOffsetLimit: 10, }, @@ -419,7 +419,7 @@ func TestServeSearch(t *testing.T) { Limit: 3, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 0f24d93..57431ed 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -9,7 +9,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) @@ -23,15 +23,11 @@ var ( ) func setupTestAPI(data test.APITestData) *API { - // when test cases don't explicitly provide configuration. - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} - } seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB for _, envConfig := range data.Cfg.Envs { - seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB + seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB } var asyncSvc asyncsearches.Service @@ -39,7 +35,7 @@ func setupTestAPI(data test.APITestData) *API { asyncSvc = data.Mocks.AsyncSearchesSvc } - return New(data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) + return New(&data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) } func withQueryParamID(h http.HandlerFunc, id string) http.HandlerFunc { diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index f32d657..bd04cb3 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -17,7 +17,7 @@ type SeqAPI struct { } func New( - cfg config.SeqAPI, + cfg *config.SeqAPI, seqDB map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index f0b2bc4..b95a703 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -52,6 +52,9 @@ func migrateServer(src *v1.Server) *v2.Server { } if src.JWTSecretKey != "" { + if dst.Auth == nil { + dst.Auth = &v2.Auth{} + } dst.Auth.JWT = &v2.JWT{SecretKey: src.JWTSecretKey} } @@ -152,7 +155,7 @@ func migrateClients(src v1.Config) *v2.Clients { if src.Server != nil && src.Server.CH != nil { ch := src.Server.CH dst.ClickHouse = []v2.CHClient{{ - ID: v2.DefaultSeqDBClientID, + ID: v2.DefaultCHClientID, Addrs: ch.Addrs, Database: ch.Database, Username: ch.Username, @@ -381,8 +384,9 @@ func migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { return dst } -func migrateErrorGroups(eg v1.ErrorGroups) v2.ErrorGroups { - return v2.ErrorGroups{ +func migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { + return &v2.ErrorGroups{ + CHID: v2.DefaultCHClientID, LogTagsMapping: v2.LogTagsMapping{ Env: eg.LogTagsMapping.Env, Service: eg.LogTagsMapping.Service, diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 286e836..8c237c2 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -28,6 +28,7 @@ import ( // addr: // auth: // oidc: +// cache_id: // skip_verify: // auth_urls: // root_ca: @@ -80,15 +81,9 @@ import ( // read_timeout: // handlers: // seq_api: -// options: -// max_search_limit: -// max_search_total_limit: -// max_search_offset_limit: -// max_export_limit: -// seq_cli_max_search_limit: -// max_parallel_export_requests: -// max_aggregations_per_request: -// max_buckets_per_aggregation_ts: +// cache_id: +// redis_id: +// global_options: // events_cache_ttl: // pinned_fields: // name: @@ -115,6 +110,15 @@ import ( // values: // process_fields: // ignore_fields: +// options: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: // envs: // : // seq_db_id: @@ -127,34 +131,9 @@ import ( // max_parallel_export_requests: // max_aggregations_per_request: // max_buckets_per_aggregation_ts: -// events_cache_ttl: -// pinned_fields: -// name: -// type: -// system_fields: -// name: -// type: -// logs_lifespan_cache_key: -// logs_lifespan_cache_ttl: -// fields_cache_ttl: -// masking: -// masks: -// re: -// groups: -// mode: -// replace_word: -// process_fields: -// ignore_fields: -// field_filters: -// condition: -// filters: -// field: -// mode: -// values: -// process_fields: -// ignore_fields: // default_env: // error_groups: +// ch_id: // log_tags_mapping: // env: // service: @@ -162,6 +141,7 @@ import ( // query_filter: // : // mass_export: +// seq_db_id: // batch_size: // workers_count: // tasks_channel_size: @@ -183,6 +163,7 @@ import ( // initial_retry_backoff: // max_retry_backoff: // async_search: +// seq_db_id: // admin_users: // list_query_length_limit: // db: @@ -213,6 +194,7 @@ import ( const ( DefaultSeqDBClientID = "default" + DefaultCHClientID = "default" DefaultInmemCacheID = "seqapi" DefaultRedisID = "default" DefaultMassExportRedisID = "mass_export" @@ -502,10 +484,10 @@ func (c *Clients) ClickHouseByID(id string) *CHClient { } type Handlers struct { - SeqAPI SeqAPI `yaml:"seq_api"` - ErrorGroups ErrorGroups `yaml:"error_groups"` - MassExport *MassExport `yaml:"mass_export"` - AsyncSearch AsyncSearch `yaml:"async_search"` + SeqAPI SeqAPI `yaml:"seq_api"` + ErrorGroups *ErrorGroups `yaml:"error_groups"` + MassExport *MassExport `yaml:"mass_export"` + AsyncSearch AsyncSearch `yaml:"async_search"` } type Field struct { @@ -723,12 +705,37 @@ func Normalize(cfg *Config) error { } if cfg.Handlers.MassExport != nil { - if cfg.Handlers.MassExport.SessionStore.RedisID == "" { - return fmt.Errorf("handlers.mass_export.session_store.redis_id cannot be empty") + if cfg.Handlers.MassExport.SeqDBID == "" { + return fmt.Errorf("handlers.mass_export.seq_db_id cannot be empty") + } + if _, ok := seqDBIDs[cfg.Handlers.MassExport.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.mass_export.seq_db_id %q", cfg.Handlers.MassExport.SeqDBID) + } + + if cfg.Handlers.MassExport.SessionStore != nil { + if cfg.Handlers.MassExport.SessionStore.RedisID == "" { + return fmt.Errorf("handlers.mass_export.session_store.redis_id cannot be empty") + } + if _, ok := redisIDs[cfg.Handlers.MassExport.SessionStore.RedisID]; !ok { + return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) + } + } + } + + if cfg.Handlers.AsyncSearch.SeqDBID == "" { + return fmt.Errorf("handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) + } + if _, ok := seqDBIDs[cfg.Handlers.AsyncSearch.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) + } + + if cfg.Handlers.ErrorGroups != nil { + if cfg.Handlers.ErrorGroups.CHID == "" { + return fmt.Errorf("handlers.error_groups.ch_id cannot be empty") } - if _, ok := redisIDs[cfg.Handlers.MassExport.SessionStore.RedisID]; !ok { - return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) + if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { + return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) } } diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index 5cf2a1c..4c7f7b1 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -4,9 +4,10 @@ import ( "context" "fmt" + "go.uber.org/zap" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/logger" - "go.uber.org/zap" ) func FromConfig(ctx context.Context, cfg *config.Cache) (map[string]Cache, error) { diff --git a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go index eeba398..0fdab5f 100644 --- a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go +++ b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: v1/seq_proxy_api.proto package seqproxyapi diff --git a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go index ee491b3..2561cd9 100644 --- a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go +++ b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: v1/seq_proxy_api.proto package seqproxyapi diff --git a/internal/pkg/service/massexport/seq_proxy_downloader.go b/internal/pkg/service/massexport/seq_proxy_downloader.go index 4a92f3e..6bb6258 100644 --- a/internal/pkg/service/massexport/seq_proxy_downloader.go +++ b/internal/pkg/service/massexport/seq_proxy_downloader.go @@ -17,13 +17,13 @@ import ( ) type seqProxyDownloader struct { - cfg config.SeqProxyDownloader + cfg config.DownloadParams client seqdb.Client } const defaultDelay = 1 * time.Second -func newSeqProxyDownloader(client seqdb.Client, cfg config.SeqProxyDownloader) *seqProxyDownloader { +func newSeqProxyDownloader(client seqdb.Client, cfg config.DownloadParams) *seqProxyDownloader { if cfg.Delay <= 0 { cfg.Delay = defaultDelay } diff --git a/internal/pkg/service/massexport/service.go b/internal/pkg/service/massexport/service.go index e65ba6b..03256c9 100644 --- a/internal/pkg/service/massexport/service.go +++ b/internal/pkg/service/massexport/service.go @@ -93,7 +93,7 @@ func NewService( tasksChannelSize = cfg.TasksChannelSize } - downloader := newSeqProxyDownloader(client, *cfg.SeqProxyDownloader) + downloader := newSeqProxyDownloader(client, *cfg.DownloadParams) return &exportService{ sessionStore: sessionStore, diff --git a/internal/pkg/service/massexport/sessionstore/redis.go b/internal/pkg/service/massexport/sessionstore/redis.go index e5a7988..ebe503e 100644 --- a/internal/pkg/service/massexport/sessionstore/redis.go +++ b/internal/pkg/service/massexport/sessionstore/redis.go @@ -31,23 +31,23 @@ const ( maxLifetime = 30 * day ) -func NewRedisSessionStore(ctx context.Context, cfg *config.SessionStore) (SessionStore, error) { - client, err := redisclient.New(ctx, &cfg.Redis) +func NewRedisSessionStore(ctx context.Context, redisCfg *config.Redis, exportLifetimeCfg time.Duration) (SessionStore, error) { + client, err := redisclient.New(ctx, redisCfg) if err != nil { return nil, fmt.Errorf("create redis client: %w", err) } exportLifetime := defaultLifetime - if cfg.ExportLifetime != 0 { + if exportLifetimeCfg != 0 { switch { - case cfg.ExportLifetime < minLifetime: - logger.Warn("export lifetime from config is too low; set min lifetime", zap.String("export_lifetime", cfg.ExportLifetime.String())) + case exportLifetimeCfg < minLifetime: + logger.Warn("export lifetime from config is too low; set min lifetime", zap.String("export_lifetime", exportLifetimeCfg.String())) exportLifetime = minLifetime - case cfg.ExportLifetime > maxLifetime: - logger.Warn("export lifetime from config is too high; set max lifetime", zap.String("export_lifetime", cfg.ExportLifetime.String())) + case exportLifetimeCfg > maxLifetime: + logger.Warn("export lifetime from config is too high; set max lifetime", zap.String("export_lifetime", exportLifetimeCfg.String())) exportLifetime = maxLifetime default: - exportLifetime = cfg.ExportLifetime + exportLifetime = exportLifetimeCfg } } diff --git a/pkg/dashboards/v1/dashboards.pb.go b/pkg/dashboards/v1/dashboards.pb.go index 7f74181..d11a515 100644 --- a/pkg/dashboards/v1/dashboards.pb.go +++ b/pkg/dashboards/v1/dashboards.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: dashboards/v1/dashboards.proto package dashboards diff --git a/pkg/dashboards/v1/dashboards_grpc.pb.go b/pkg/dashboards/v1/dashboards_grpc.pb.go index 8217c47..9559155 100644 --- a/pkg/dashboards/v1/dashboards_grpc.pb.go +++ b/pkg/dashboards/v1/dashboards_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: dashboards/v1/dashboards.proto package dashboards diff --git a/pkg/errorgroups/v1/errorgroups.pb.go b/pkg/errorgroups/v1/errorgroups.pb.go index cc0e2c1..763796e 100644 --- a/pkg/errorgroups/v1/errorgroups.pb.go +++ b/pkg/errorgroups/v1/errorgroups.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: errorgroups/v1/errorgroups.proto package errorgroups diff --git a/pkg/errorgroups/v1/errorgroups_grpc.pb.go b/pkg/errorgroups/v1/errorgroups_grpc.pb.go index f8c6f27..a284026 100644 --- a/pkg/errorgroups/v1/errorgroups_grpc.pb.go +++ b/pkg/errorgroups/v1/errorgroups_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: errorgroups/v1/errorgroups.proto package errorgroups diff --git a/pkg/massexport/v1/massexport.pb.go b/pkg/massexport/v1/massexport.pb.go index 47c9007..3c5f3e9 100644 --- a/pkg/massexport/v1/massexport.pb.go +++ b/pkg/massexport/v1/massexport.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: massexport/v1/massexport.proto package massexport diff --git a/pkg/massexport/v1/massexport_grpc.pb.go b/pkg/massexport/v1/massexport_grpc.pb.go index eaaea70..a78e188 100644 --- a/pkg/massexport/v1/massexport_grpc.pb.go +++ b/pkg/massexport/v1/massexport_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: massexport/v1/massexport.proto package massexport diff --git a/pkg/seqapi/v1/seq_api.pb.go b/pkg/seqapi/v1/seq_api.pb.go index 2e5fa5b..c22da34 100644 --- a/pkg/seqapi/v1/seq_api.pb.go +++ b/pkg/seqapi/v1/seq_api.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: seqapi/v1/seq_api.proto package seqapi diff --git a/pkg/seqapi/v1/seq_api_grpc.pb.go b/pkg/seqapi/v1/seq_api_grpc.pb.go index 83a451d..e6a5155 100644 --- a/pkg/seqapi/v1/seq_api_grpc.pb.go +++ b/pkg/seqapi/v1/seq_api_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: seqapi/v1/seq_api.proto package seqapi diff --git a/pkg/userprofile/v1/userprofile.pb.go b/pkg/userprofile/v1/userprofile.pb.go index d8e85f0..7f283ae 100644 --- a/pkg/userprofile/v1/userprofile.pb.go +++ b/pkg/userprofile/v1/userprofile.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: userprofile/v1/userprofile.proto package userprofile diff --git a/pkg/userprofile/v1/userprofile_grpc.pb.go b/pkg/userprofile/v1/userprofile_grpc.pb.go index a2c1330..93ce310 100644 --- a/pkg/userprofile/v1/userprofile_grpc.pb.go +++ b/pkg/userprofile/v1/userprofile_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: userprofile/v1/userprofile.proto package userprofile From 57cfb62d3c2810b8b984cbf9dd235c255acabbe1 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Fri, 31 Jul 2026 12:25:23 +0300 Subject: [PATCH 09/29] refactor --- Makefile | 5 + cmd/config-migrate/main.go | 59 +++++-- cmd/seq-ui/main.go | 6 +- internal/api/errorgroups/v1/errorgroups.go | 3 +- internal/api/seqapi/v1/grpc/api.go | 13 +- internal/api/seqapi/v1/grpc/logs_lifespan.go | 5 +- .../api/seqapi/v1/grpc/logs_lifespan_test.go | 1 - internal/api/seqapi/v1/grpc/test_data.go | 4 + internal/api/seqapi/v1/http/api.go | 13 +- internal/api/seqapi/v1/http/logs_lifespan.go | 5 +- .../api/seqapi/v1/http/logs_lifespan_test.go | 1 - internal/api/seqapi/v1/http/test_data.go | 4 + internal/app/auth/oidc.go | 28 ++-- .../config/{config.go => loader/loader.go} | 27 ++- internal/app/config/migrate/v1_v2.go | 117 ++++++++++--- internal/app/config/v1/config.go | 1 - internal/app/config/v2/config.go | 157 ++++++++++++++---- 17 files changed, 314 insertions(+), 135 deletions(-) rename internal/app/config/{config.go => loader/loader.go} (79%) diff --git a/Makefile b/Makefile index f236185..3332e47 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ TIME := $(shell date '+%Y-%m-%d_%H:%M:%S') MIGRATION_DSN ?= postgresql://localhost/postgres?sslmode=disable&user=postgres&password=postgres MIGRATION_DSN_CLICKHOUSE ?= tcp://default@localhost:9000/seq_ui_server +SOURCE_CONFIG ?= config/config.local.yaml LOCAL_BIN := $(CURDIR)/bin @@ -58,6 +59,10 @@ push-migration-image: build-migration-image run: .check-config go run ./cmd/seq-ui -config=config/config.local.yaml +.PHONY: config-migrate +config-migrate: + go run ./cmd/config-migrate -source=${SOURCE_CONFIG} + .PHONY: .check-config .check-config: @if [ ! -f config/config.local.yaml ]; then\ diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go index 3dd1f2c..da63143 100644 --- a/cmd/config-migrate/main.go +++ b/cmd/config-migrate/main.go @@ -1,13 +1,19 @@ package main import ( - "errors" + "bytes" "flag" + "fmt" "os" + "path/filepath" + "strings" "go.uber.org/zap" + "gopkg.in/yaml.v3" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/loader" + "github.com/ozontech/seq-ui/internal/app/config/migrate" + v1 "github.com/ozontech/seq-ui/internal/app/config/v1" "github.com/ozontech/seq-ui/logger" ) @@ -26,24 +32,55 @@ func run(source string) { logger.Fatal("missing required parameter", zap.String("param", "-source")) } - legacyCfg, err := os.ReadFile(source) + stat, err := os.Stat(source) if err != nil { - logger.Fatal("error reading legacy config", zap.String("source", source), zap.Error(err)) + logger.Fatal("stat source", zap.String("source", source), zap.Error(err)) } - backupPath := source + ".bak" + sourceCfg, err := os.ReadFile(source) //nolint:gosec + if err != nil { + logger.Fatal("error reading source config", zap.String("source", source), zap.Error(err)) + } + version, err := loader.ReadVersion(sourceCfg) + if err != nil { + logger.Fatal("read config version", zap.Error(err)) + } + if version != loader.V1 { + errStr := fmt.Sprintf("source config is not v%d, nothing to migrate", loader.V1) + logger.Fatal(errStr, zap.Int("version", version)) + } + + backupPath := strings.TrimSuffix(source, filepath.Ext(source)) + ".bck" + filepath.Ext(source) if _, err := os.Stat(backupPath); err == nil { logger.Fatal("backup already exists, remove it before re-running", zap.String("backup", backupPath)) - } else if !errors.Is(err, os.ErrNotExist) { - logger.Fatal("stat backup", zap.String("backup", backupPath), zap.Error(err)) } - if err := os.WriteFile(backupPath, legacyCfg, 0644); err != nil { + if err := os.WriteFile(backupPath, sourceCfg, stat.Mode()); err != nil { logger.Fatal("error writing backup legacy config", zap.String("backup", backupPath), zap.Error(err)) } - backupCfg, err := config.FromFile(backupPath) - if err != nil { - logger.Fatal("read backup config file error", zap.Error(err)) + var cfg v1.Config + decoder := yaml.NewDecoder(bytes.NewReader(sourceCfg)) + decoder.KnownFields(true) + if err := decoder.Decode(&cfg); err != nil { + logger.Fatal("error parsing config v1", zap.Error(err)) } + + migratedCfg := migrate.V1ToV2(cfg) + + var buf bytes.Buffer + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + if err := encoder.Encode(&migratedCfg); err != nil { + logger.Fatal("error encoding config v2", zap.Error(err)) + } + if err := encoder.Close(); err != nil { + logger.Fatal("close encoder", zap.Error(err)) + } + + if err := os.WriteFile(source, buf.Bytes(), stat.Mode()); err != nil { + logger.Fatal("write migrated config", zap.Error(err)) + } + + logger.Info("config migrated", zap.String("source", source), zap.String("backup", backupPath)) } diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index fdadaeb..1139e5f 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -23,7 +23,7 @@ import ( massexport_v1 "github.com/ozontech/seq-ui/internal/api/massexport/v1" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" - configloader "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/loader" config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/server" "github.com/ozontech/seq-ui/internal/pkg/cache" @@ -75,7 +75,7 @@ func run(ctx context.Context) { logger.Warn("app uses the default config file, to provide your own config use -config flag") } - cfg, err := configloader.FromFile(*configPath) + cfg, err := loader.FromFile(*configPath) if err != nil { logger.Fatal("read config file error", zap.Error(err)) } @@ -174,7 +174,7 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac repo := repositorych.New(chClients[cfg.Handlers.ErrorGroups.CHID], chCfg.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) svc := errorgroups.New(repo, cfg.Handlers.ErrorGroups.LogTagsMapping) - errorGroupsV1 = errorgroups_v1.New(svc) + errorGroupsV1 = errorgroups_v1.New(svc, chClients) } return api.NewRegistrar(seqApiV1, userProfileV1, dashboardsV1, massExportV1, errorGroupsV1) diff --git a/internal/api/errorgroups/v1/errorgroups.go b/internal/api/errorgroups/v1/errorgroups.go index dd3f341..91af47a 100644 --- a/internal/api/errorgroups/v1/errorgroups.go +++ b/internal/api/errorgroups/v1/errorgroups.go @@ -1,6 +1,7 @@ package errorgroups_v1 import ( + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "github.com/go-chi/chi/v5" grpc_api "github.com/ozontech/seq-ui/internal/api/errorgroups/v1/grpc" @@ -13,7 +14,7 @@ type ErrorGroups struct { httpAPI *http_api.API } -func New(svc errorgroups.Service) *ErrorGroups { +func New(svc errorgroups.Service, chClients map[string]driver.Conn) *ErrorGroups { return &ErrorGroups{ grpcAPI: grpc_api.New(svc), httpAPI: http_api.New(svc), diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index cc75ad0..dcdb7e8 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -31,7 +31,6 @@ type apiGlobalParams struct { pinnedFields []*seqapi.Field systemFields []*seqapi.Field eventsCacheTTL time.Duration - logsLifespanCacheKey string logsLifespanCacheTTL time.Duration } @@ -59,7 +58,6 @@ func New( pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), systemFields: parseFields(cfg.GlobalOptions.SystemFields), eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, - logsLifespanCacheKey: cfg.GlobalOptions.LogsLifespanCacheKey, logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, } @@ -82,17 +80,8 @@ func New( } } } else { - if len(seqDBСlients) != 1 { - logger.Fatal("seq_api.envs is empty, configure envs when clients.seq_db has more than one instance") - } - - var client seqdb.Client - for _, c := range seqDBСlients { - client = c - } - paramsByEnv[""] = apiParams{ - client: client, + client: seqDBСlients[cfg.SeqDBID], options: &cfg.Options, } } diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan.go b/internal/api/seqapi/v1/grpc/logs_lifespan.go index 09ed1fc..ed3dec8 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan.go @@ -30,7 +30,8 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ return nil, status.Error(codes.InvalidArgument, err.Error()) } - if countStr, err := a.redisCache.Get(ctx, a.globalParams.logsLifespanCacheKey); err == nil { + cacheKey := "logs_lifespan" + if countStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { count := 0 count, err = strconv.Atoi(countStr) if err == nil { @@ -56,7 +57,7 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ count := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) res := time.Duration(count) * lifespan.MeasureUnit - err = a.redisCache.SetWithTTL(ctx, a.globalParams.logsLifespanCacheKey, strconv.Itoa(count), a.globalParams.logsLifespanCacheTTL) + err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(count), a.globalParams.logsLifespanCacheTTL) if err != nil { logger.Error("can't set logs lifespan to cache", zap.Error(err)) } diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index a82cb88..cf4ddff 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -107,7 +107,6 @@ func TestGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ GlobalOptions: config.SeqAPIGlobalOptions{ - LogsLifespanCacheKey: cacheKey, LogsLifespanCacheTTL: cacheTTL, }, }, diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index ce8ed4e..b7464e6 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -18,6 +18,10 @@ var ( ) func setupTestAPI(data test.APITestData) *API { + if len(data.Cfg.Envs) == 0 && data.Cfg.SeqDBID == "" { + data.Cfg.SeqDBID = config.DefaultSeqDBClientID + } + seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index 2f5b4b0..be45efa 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -33,7 +33,6 @@ type apiGlobalParams struct { pinnedFields fields systemFields fields eventsCacheTTL time.Duration - logsLifespanCacheKey string logsLifespanCacheTTL time.Duration } @@ -59,7 +58,6 @@ func New( pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), systemFields: parseFields(cfg.GlobalOptions.SystemFields), eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, - logsLifespanCacheKey: cfg.GlobalOptions.LogsLifespanCacheKey, logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, } @@ -85,17 +83,8 @@ func New( } } } else { - if len(seqDBСlients) != 1 { - logger.Fatal("seq_api.envs is empty, configure envs when clients.seq_db has more than one instance") - } - - var client seqdb.Client - for _, c := range seqDBСlients { - client = c - } - paramsByEnv[""] = apiParams{ - client: client, + client: seqDBСlients[cfg.SeqDBID], options: &cfg.Options, exportLimiter: globalExportLimiter, } diff --git a/internal/api/seqapi/v1/http/logs_lifespan.go b/internal/api/seqapi/v1/http/logs_lifespan.go index 9e52e55..8d68811 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan.go +++ b/internal/api/seqapi/v1/http/logs_lifespan.go @@ -37,7 +37,8 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { return } - if resStr, err := a.redisCache.Get(ctx, a.globalParams.logsLifespanCacheKey); err == nil { + cacheKey := "logs_lifespan" + if resStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { res := 0 res, err = strconv.Atoi(resStr) if err == nil { @@ -64,7 +65,7 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { res := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) - err = a.redisCache.SetWithTTL(ctx, a.globalParams.logsLifespanCacheKey, strconv.Itoa(res), a.globalParams.logsLifespanCacheTTL) + err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(res), a.globalParams.logsLifespanCacheTTL) if err != nil { logger.Error("can't set logs lifespan to cache", zap.Error(err)) } diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index 692ef86..bd36447 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -104,7 +104,6 @@ func TestServeGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ GlobalOptions: config.SeqAPIGlobalOptions{ - LogsLifespanCacheKey: cacheKey, LogsLifespanCacheTTL: cacheTTL, }, }, diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 57431ed..efab1f0 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -23,6 +23,10 @@ var ( ) func setupTestAPI(data test.APITestData) *API { + if len(data.Cfg.Envs) == 0 && data.Cfg.SeqDBID == "" { + data.Cfg.SeqDBID = config.DefaultSeqDBClientID + } + seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 4a53756..498c24b 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -49,15 +49,7 @@ func NewOIDCProvider(ctx context.Context, cfg *config.OIDC, oidcCache cache.Cach ) if !cfg.SkipVerify { - oidcCtx, err = newHTTPContext( - context.Background(), - httpContextCfg{ - rootCA: cfg.RootCA, - caCert: cfg.CACert, - privateKey: cfg.PrivateKey, - sslSkipVerify: cfg.SSLSkipVerify, - }, - ) + oidcCtx, err = newHTTPContext(context.Background(), cfg.TLS) if err != nil { return nil, err } @@ -176,24 +168,24 @@ type httpContextCfg struct { sslSkipVerify bool } -func newHTTPContext(ctx context.Context, conf httpContextCfg) (context.Context, error) { +func newHTTPContext(ctx context.Context, cfg *config.TLS) (context.Context, error) { hc := &http.Client{ Timeout: oidcClientTimeout, } - if conf.isZero() { + if cfg == nil { return oidc.ClientContext(ctx, hc), nil } b := tls.NewConfigBuilder() - if conf.sslSkipVerify { - b.SetInsecureSkipVerify(conf.sslSkipVerify) - } else if !conf.sslSkipVerify && (conf.rootCA != "" || conf.caCert != "" || conf.privateKey != "") { - if conf.rootCA != "" { - if err := b.AppendCARoot(conf.rootCA); err != nil { + if cfg.Insecure { + b.SetInsecureSkipVerify(cfg.Insecure) + } else if !cfg.Insecure && (cfg.RootCA != "" || cfg.CACert != "" || cfg.PrivateKey != "") { + if cfg.RootCA != "" { + if err := b.AppendCARoot(cfg.RootCA); err != nil { return nil, fmt.Errorf("can't append CA root: %w", err) } } - if conf.caCert != "" || conf.privateKey != "" { - if err := b.AppendX509KeyPair(conf.caCert, conf.privateKey); err != nil { + if cfg.CACert != "" || cfg.PrivateKey != "" { + if err := b.AppendX509KeyPair(cfg.CACert, cfg.PrivateKey); err != nil { return nil, fmt.Errorf("can't append key pair: %w", err) } } diff --git a/internal/app/config/config.go b/internal/app/config/loader/loader.go similarity index 79% rename from internal/app/config/config.go rename to internal/app/config/loader/loader.go index 78e7638..81401ad 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/loader/loader.go @@ -1,4 +1,4 @@ -package config +package loader import ( "bytes" @@ -13,8 +13,8 @@ import ( ) const ( - previousVersion = 1 - currentVersion = 2 + V1 = 1 + V2 = 2 ) type configMeta struct { @@ -35,13 +35,13 @@ func FromFile(cfgPath string) (v2.Config, error) { cfg := v2.Config{} switch { - case meta.Version == nil || *meta.Version == previousVersion: + case meta.Version == nil || *meta.Version == V1: cfgV1, err := parse[v1.Config](cfgBytes, true) if err != nil { return v2.Config{}, err } cfg = migrate.V1ToV2(cfgV1) - case *meta.Version == currentVersion: + case *meta.Version == V2: cfg, err = parse[v2.Config](cfgBytes, true) if err != nil { return v2.Config{}, err @@ -57,13 +57,24 @@ func FromFile(cfgPath string) (v2.Config, error) { return cfg, nil } +func ReadVersion(cfgBytes []byte) (int, error) { + meta, err := parse[configMeta](cfgBytes, false) + if err != nil { + return 0, err + } + + if meta.Version == nil { + return V1, nil + } + + return *meta.Version, nil +} + func parse[T any](cfg []byte, strict bool) (T, error) { var result T decoder := yaml.NewDecoder(bytes.NewReader(cfg)) - if strict { - decoder.KnownFields(true) - } + decoder.KnownFields(strict) if err := decoder.Decode(&result); err != nil { return result, fmt.Errorf("error parsing config: %w", err) } diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index b95a703..2d4e73e 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -12,7 +12,7 @@ func V1ToV2(src v1.Config) v2.Config { dst.Cache = migrateCache(src.Server) dst.Clients = migrateClients(src) dst.DB = migrateDB(src.Server) - dst.Handlers = migrateHandlers(src.Handlers, &dst) + dst.Handlers = migrateHandlers(src.Handlers, &dst, src.Server.Cache) return dst } @@ -37,18 +37,19 @@ func migrateServer(src *v1.Server) *v2.Server { } if src.OIDC != nil { - dst.Auth = &v2.Auth{ - OIDC: &v2.OIDC{ - SkipVerify: src.OIDC.SkipVerify, - AuthURLs: src.OIDC.AuthURLs, - RootCA: src.OIDC.RootCA, - CACert: src.OIDC.CACert, - PrivateKey: src.OIDC.PrivateKey, - SSLSkipVerify: src.OIDC.SSLSkipVerify, - AllowedClients: src.OIDC.AllowedClients, - CacheSecretKey: src.OIDC.CacheSecretKey, - }, + oidc := &v2.OIDC{ + SkipVerify: src.OIDC.SkipVerify, + AuthURLs: src.OIDC.AuthURLs, + TLS: migrateTLS(src.OIDC), + AllowedClients: src.OIDC.AllowedClients, + } + + if src.OIDC.CacheSecretKey != "" { + cacheID, _ := defaultCacheIDs(src.Cache) + oidc.CacheID = cacheID } + + dst.Auth = &v2.Auth{OIDC: oidc} } if src.JWTSecretKey != "" { @@ -61,6 +62,20 @@ func migrateServer(src *v1.Server) *v2.Server { return dst } +func migrateTLS(src *v1.OIDC) *v2.TLS { + if src.RootCA == "" && src.CACert == "" && + src.PrivateKey == "" && !src.SSLSkipVerify { + return nil + } + + return &v2.TLS{ + RootCA: src.RootCA, + CACert: src.CACert, + PrivateKey: src.PrivateKey, + Insecure: src.SSLSkipVerify, + } +} + func migrateCORS(src *v1.CORS) *v2.CORS { if src == nil { return nil @@ -134,8 +149,8 @@ func migrateClients(src v1.Config) *v2.Clients { if src.Clients != nil { if len(src.Clients.SeqDB) > 0 { dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) - for _, s := range src.Clients.SeqDB { - dst.SeqDB = append(dst.SeqDB, migrateSeqDBClient(s)) + for i := range src.Clients.SeqDB { + dst.SeqDB = append(dst.SeqDB, migrateSeqDBClient(&src.Clients.SeqDB[i])) } } else { dst.SeqDB = []v2.SeqDBClient{{ @@ -169,7 +184,7 @@ func migrateClients(src v1.Config) *v2.Clients { return dst } -func migrateSeqDBClient(src v1.SeqDBClient) v2.SeqDBClient { +func migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { return v2.SeqDBClient{ ID: src.ID, Timeout: src.Timeout, @@ -206,15 +221,25 @@ func migrateCache(src *v1.Server) *v2.Cache { } cache := &v2.Cache{} - cache.Inmemory = append(cache.Inmemory, v2.InmemoryCache{ - ID: v2.DefaultInmemCacheID, - NumCounters: src.Cache.Inmemory.NumCounters, - MaxCost: src.Cache.Inmemory.MaxCost, - BufferItems: src.Cache.Inmemory.BufferItems, - }) + + if !isInmemEmpty(src.Cache.Inmemory) { + cache.Inmemory = append(cache.Inmemory, v2.InmemoryCache{ + ID: v2.DefaultInmemCacheID, + NumCounters: src.Cache.Inmemory.NumCounters, + MaxCost: src.Cache.Inmemory.MaxCost, + BufferItems: src.Cache.Inmemory.BufferItems, + }) + } if src.Cache.Redis != nil { - cache.Redis = append(cache.Redis, migrateRedis(src.Cache.Redis, v2.DefaultRedisID, v2.DefaultInmemCacheID)) + if len(cache.Inmemory) > 0 { + cache.Redis = append(cache.Redis, + migrateRedis(src.Cache.Redis, v2.DefaultRedisID, v2.DefaultInmemCacheID), + migrateRedis(src.Cache.Redis, v2.DefaultRedis2ID, ""), + ) + } else { + cache.Redis = append(cache.Redis, migrateRedis(src.Cache.Redis, v2.DefaultRedisID, "")) + } } return cache @@ -234,13 +259,13 @@ func migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { } } -func migrateHandlers(src *v1.Handlers, cfg *v2.Config) *v2.Handlers { +func migrateHandlers(src *v1.Handlers, cfg *v2.Config, v1Cache v1.Cache) *v2.Handlers { if src == nil { return &v2.Handlers{} } dst := &v2.Handlers{ - SeqAPI: migrateSeqAPI(src.SeqAPI), + SeqAPI: migrateSeqAPI(src.SeqAPI, v1Cache), ErrorGroups: migrateErrorGroups(src.ErrorGroups), AsyncSearch: migrateAsyncSearch(src.AsyncSearch), } @@ -252,12 +277,19 @@ func migrateHandlers(src *v1.Handlers, cfg *v2.Config) *v2.Handlers { return dst } -func migrateSeqAPI(src v1.SeqAPI) v2.SeqAPI { +func migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { + cacheID, redisID := defaultCacheIDs(v1Cache) dst := v2.SeqAPI{ + CacheID: cacheID, + RedisID: redisID, Envs: migrateSeqAPIEnvs(src.Envs), DefaultEnv: src.DefaultEnv, } + if len(src.Envs) == 0 { + dst.SeqDBID = v2.DefaultSeqDBClientID + } + if src.SeqAPIOptions != nil { dst.GlobalOptions = migrateSeqAPIOptionsToGlobal(src.SeqAPIOptions) dst.Options = migrateSeqAPIOptions(src.SeqAPIOptions) @@ -292,7 +324,6 @@ func migrateSeqAPIOptionsToGlobal(options *v1.SeqAPIOptions) v2.SeqAPIGlobalOpti EventsCacheTTL: options.EventsCacheTTL, PinnedFields: migrateFields(options.PinnedFields), SystemFields: migrateFields(options.SystemFields), - LogsLifespanCacheKey: options.LogsLifespanCacheKey, LogsLifespanCacheTTL: options.LogsLifespanCacheTTL, FieldsCacheTTL: options.FieldsCacheTTL, Masking: migrateMasking(options.Masking), @@ -346,7 +377,8 @@ func migrateMasks(ms []v1.Mask) []v2.Mask { } dst := make([]v2.Mask, len(ms)) - for i, m := range ms { + for i := range ms { + m := &ms[i] dst[i] = v2.Mask{ Re: m.Re, Groups: m.Groups, @@ -385,6 +417,10 @@ func migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { } func migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { + if isEgEmpty(eg) { + return nil + } + return &v2.ErrorGroups{ CHID: v2.DefaultCHClientID, LogTagsMapping: v2.LogTagsMapping{ @@ -398,6 +434,7 @@ func migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { func migrateAsyncSearch(a v1.AsyncSearch) v2.AsyncSearch { return v2.AsyncSearch{ + SeqDBID: v2.DefaultSeqDBClientID, AdminUsers: a.AdminUsers, ListQueryLengthLimit: a.ListQueryLengthLimit, } @@ -405,6 +442,7 @@ func migrateAsyncSearch(a v1.AsyncSearch) v2.AsyncSearch { func migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { dst := &v2.MassExport{ + SeqDBID: v2.DefaultSeqDBClientID, BatchSize: me.BatchSize, WorkersCount: me.WorkersCount, TasksChannelSize: me.TasksChannelSize, @@ -457,3 +495,28 @@ func migrateDownloadParams(src *v1.SeqProxyDownloader) *v2.DownloadParams { MaxRetryBackoff: src.MaxRetryBackoff, } } + +func defaultCacheIDs(src v1.Cache) (cacheID, redisID string) { + hasInmem := !isInmemEmpty(src.Inmemory) + hasRedis := src.Redis != nil + + switch { + case hasInmem && hasRedis: + return v2.DefaultRedisID, v2.DefaultRedis2ID + case hasRedis: + return v2.DefaultRedisID, v2.DefaultRedisID + case hasInmem: + return v2.DefaultInmemCacheID, "" + default: + return "", "" + } +} + +func isInmemEmpty(src v1.InmemoryCache) bool { + return src.BufferItems == 0 && src.MaxCost == 0 && src.NumCounters == 0 +} + +func isEgEmpty(src v1.ErrorGroups) bool { + return len(src.LogTagsMapping.Env) == 0 && len(src.LogTagsMapping.Service) == 0 && + len(src.LogTagsMapping.Release) == 0 && len(src.QueryFilter) == 0 +} diff --git a/internal/app/config/v1/config.go b/internal/app/config/v1/config.go index e3deb74..a56a6fa 100644 --- a/internal/app/config/v1/config.go +++ b/internal/app/config/v1/config.go @@ -5,7 +5,6 @@ import ( ) // Deprecated configuration scheme -// version: // server: // http_addr: // grpc_addr: diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 8c237c2..49f1a06 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -29,14 +29,15 @@ import ( // auth: // oidc: // cache_id: +// cache_secret_key: // skip_verify: // auth_urls: -// root_ca: -// ca_cert: -// private_key: -// ssl_skip_verify: +// tls: +// root_ca: +// ca_cert: +// private_key: +// insecure: // allowed_clients: -// cache_secret_key: // jwt: // secret_key: // rate_limiters: @@ -81,6 +82,7 @@ import ( // read_timeout: // handlers: // seq_api: +// seq_db_id: // cache_id: // redis_id: // global_options: @@ -91,7 +93,6 @@ import ( // system_fields: // name: // type: -// logs_lifespan_cache_key: // logs_lifespan_cache_ttl: // fields_cache_ttl: // masking: @@ -134,6 +135,10 @@ import ( // default_env: // error_groups: // ch_id: +// envs: +// : +// ch_id: +// default_env: // log_tags_mapping: // env: // service: @@ -164,6 +169,11 @@ import ( // max_retry_backoff: // async_search: // seq_db_id: +// envs: +// : +// seq_db_id: +// list_query_length_limit: +// default_env: // admin_users: // list_query_length_limit: // db: @@ -193,10 +203,11 @@ import ( // max_retry_backoff: const ( - DefaultSeqDBClientID = "default" - DefaultCHClientID = "default" - DefaultInmemCacheID = "seqapi" - DefaultRedisID = "default" + DefaultSeqDBClientID = "defaultSeqDB" + DefaultCHClientID = "defaultCH" + DefaultInmemCacheID = "defaultInmemCache" + DefaultRedisID = "defaultRedis" + DefaultRedis2ID = "defaultRedis2" DefaultMassExportRedisID = "mass_export" ProxyClientModeGRPC = "grpc" @@ -232,7 +243,6 @@ const ( defaultEventsCacheTTL = 24 * time.Hour - defaultLogsLifespanCacheKey = "logs_lifespan" defaultLogsLifespanCacheTTL = 10 * time.Minute defaultClickHouseDialTimeout = 5 * time.Second @@ -258,16 +268,20 @@ type CORS struct { OptionsPassthrough bool `yaml:"options_passthrough"` } +type TLS struct { + RootCA string `yaml:"root_ca"` + CACert string `yaml:"ca_cert"` + PrivateKey string `yaml:"private_key"` + Insecure bool `yaml:"insecure"` +} + type OIDC struct { CacheID string `yaml:"cache_id"` + CacheSecretKey string `yaml:"cache_secret_key"` SkipVerify bool `yaml:"skip_verify"` AuthURLs []string `yaml:"auth_urls"` - RootCA string `yaml:"root_ca"` - CACert string `yaml:"ca_cert"` - PrivateKey string `yaml:"private_key"` - SSLSkipVerify bool `yaml:"ssl_skip_verify"` + TLS *TLS `yaml:"tls"` AllowedClients []string `yaml:"allowed_clients"` - CacheSecretKey string `yaml:"cache_secret_key"` } type DB struct { @@ -496,6 +510,7 @@ type Field struct { } type SeqAPI struct { + SeqDBID string `yaml:"seq_db_id"` CacheID string `yaml:"cache_id"` RedisID string `yaml:"redis_id"` GlobalOptions SeqAPIGlobalOptions `yaml:"global_options"` @@ -508,7 +523,6 @@ type SeqAPIGlobalOptions struct { PinnedFields []Field `yaml:"pinned_fields"` SystemFields []Field `yaml:"system_fields"` Masking *Masking `yaml:"masking"` - LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` @@ -566,15 +580,28 @@ type LogTagsMapping struct { } type ErrorGroups struct { - CHID string `yaml:"ch_id"` - LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` - QueryFilter map[string]string `yaml:"query_filter"` + CHID string `yaml:"ch_id"` + Envs map[string]ErrorGroupsEnv `yaml:"envs"` + LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` + QueryFilter map[string]string `yaml:"query_filter"` + DefaultEnv string `yaml:"default_env"` +} + +type ErrorGroupsEnv struct { + CHID string `yaml:"ch_id"` } type AsyncSearch struct { - SeqDBID string `yaml:"seq_db_id"` - AdminUsers []string `yaml:"admin_users"` - ListQueryLengthLimit int `yaml:"list_query_length_limit"` + SeqDBID string `yaml:"seq_db_id"` + Envs map[string]AsyncSearchEnv `yaml:"envs"` + AdminUsers []string `yaml:"admin_users"` + ListQueryLengthLimit int `yaml:"list_query_length_limit"` + DefaultEnv string `yaml:"default_env"` +} + +type AsyncSearchEnv struct { + SeqDBID string `yaml:"seq_db_id"` + ListQueryLengthLimit int `yaml:"list_query_length_limit"` } func Normalize(cfg *Config) error { @@ -685,6 +712,10 @@ func Normalize(cfg *Config) error { setSeqAPIOptionsDefaults(&cfg.Handlers.SeqAPI.Options) if len(cfg.Handlers.SeqAPI.Envs) > 0 { + if cfg.Handlers.SeqAPI.SeqDBID != "" { + return fmt.Errorf("seq_api.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + } + if cfg.Handlers.SeqAPI.DefaultEnv == "" { return fmt.Errorf("default_env must be specified when using envs") } @@ -702,6 +733,14 @@ func Normalize(cfg *Config) error { envConfig.Options = &merged cfg.Handlers.SeqAPI.Envs[envName] = envConfig } + } else { + if cfg.Handlers.SeqAPI.SeqDBID == "" { + return fmt.Errorf("seq_api.seq_db_id must be specified when envs is empty") + } + + if _, ok := seqDBIDs[cfg.Handlers.SeqAPI.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.seq_api.seq_db_id %q", cfg.Handlers.SeqAPI.SeqDBID) + } } if cfg.Handlers.MassExport != nil { @@ -722,20 +761,69 @@ func Normalize(cfg *Config) error { } } - if cfg.Handlers.AsyncSearch.SeqDBID == "" { - return fmt.Errorf("handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) - } - if _, ok := seqDBIDs[cfg.Handlers.AsyncSearch.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) + if len(cfg.Handlers.AsyncSearch.Envs) > 0 { + if cfg.Handlers.AsyncSearch.SeqDBID != "" { + return fmt.Errorf("async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + } + + if cfg.Handlers.AsyncSearch.DefaultEnv == "" { + return fmt.Errorf("default_env must be specified when using envs") + } + + if _, exists := cfg.Handlers.AsyncSearch.Envs[cfg.Handlers.AsyncSearch.DefaultEnv]; !exists { + return fmt.Errorf("default_env '%s' not found in async_search.envs", cfg.Handlers.AsyncSearch.DefaultEnv) + } + + for envName, envConfig := range cfg.Handlers.AsyncSearch.Envs { + if envConfig.SeqDBID == "" { + return fmt.Errorf("handlers.async_search.envs[%q].seq_db_id cannot be empty", envName) + } + + if _, ok := seqDBIDs[cfg.Handlers.AsyncSearch.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + } + } + } else { + if cfg.Handlers.AsyncSearch.SeqDBID == "" { + return fmt.Errorf("handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) + } + if _, ok := seqDBIDs[cfg.Handlers.AsyncSearch.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) + } } if cfg.Handlers.ErrorGroups != nil { - if cfg.Handlers.ErrorGroups.CHID == "" { - return fmt.Errorf("handlers.error_groups.ch_id cannot be empty") - } + eg := cfg.Handlers.ErrorGroups + if len(eg.Envs) > 0 { + if eg.CHID != "" { + return fmt.Errorf("handlers.error_groups.ch_id must be empty when envs is used. Put ch_id inside each env") + } + + if eg.DefaultEnv == "" { + return fmt.Errorf("default_env must be specified when using envs") + } + + if _, exists := eg.Envs[eg.DefaultEnv]; !exists { + return fmt.Errorf("default_env '%s' not found in error_groups.envs", eg.DefaultEnv) + } - if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { - return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) + for envName, envConfig := range eg.Envs { + if envConfig.CHID == "" { + return fmt.Errorf("client '%s' for env '%s' not found", envConfig.CHID, envName) + } + + if _, ok := chIDs[envConfig.CHID]; !ok { + return fmt.Errorf("unknown handlers.error_groups.ch_id %q", envConfig.CHID) + } + } + } else { + if cfg.Handlers.ErrorGroups.CHID == "" { + return fmt.Errorf("handlers.error_groups.ch_id cannot be empty") + } + + if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { + return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) + } } } @@ -818,9 +906,6 @@ func setSeqAPIGlobalOptionsDefaults(options *SeqAPIGlobalOptions) { if options.EventsCacheTTL <= 0 { options.EventsCacheTTL = defaultEventsCacheTTL } - if options.LogsLifespanCacheKey == "" { - options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey - } if options.LogsLifespanCacheTTL <= 0 { options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL } From 3f452b95172128024751611e83051720e9ad3f39 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Fri, 31 Jul 2026 12:28:06 +0300 Subject: [PATCH 10/29] fix lint --- internal/app/auth/oidc.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 498c24b..0dbfb24 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -161,13 +161,6 @@ func (p *oidcProvider) checkClients(clients []string) error { return fmt.Errorf("no allowed client found (clients: %v; allowed clients: %v)", clients, p.allowedClients) } -type httpContextCfg struct { - rootCA string - caCert string - privateKey string - sslSkipVerify bool -} - func newHTTPContext(ctx context.Context, cfg *config.TLS) (context.Context, error) { hc := &http.Client{ Timeout: oidcClientTimeout, @@ -196,10 +189,6 @@ func newHTTPContext(ctx context.Context, cfg *config.TLS) (context.Context, erro return oidc.ClientContext(ctx, hc), nil } -func (c httpContextCfg) isZero() bool { - return c.rootCA == "" && c.caCert == "" && c.privateKey == "" && !c.sslSkipVerify -} - func hashToken(token, secret string) string { hash := sha512.New() _, _ = hash.Write([]byte(token + secret)) From aef2130275a3031f2fbc1d82a86d7de7927b79f3 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 5 Aug 2026 15:58:20 +0300 Subject: [PATCH 11/29] fix --- cmd/config-migrate/main.go | 11 ++- cmd/seq-ui/main.go | 3 +- internal/app/config/loader/loader.go | 20 +++--- internal/app/config/migrate/v1_v2.go | 38 ++++++---- internal/app/config/v1/config.go | 1 + internal/app/config/v2/config.go | 101 ++++++++++++++------------- internal/app/server/init.go | 2 +- internal/pkg/cache/ctor.go | 4 +- 8 files changed, 98 insertions(+), 82 deletions(-) diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go index da63143..97583e7 100644 --- a/cmd/config-migrate/main.go +++ b/cmd/config-migrate/main.go @@ -39,7 +39,7 @@ func run(source string) { sourceCfg, err := os.ReadFile(source) //nolint:gosec if err != nil { - logger.Fatal("error reading source config", zap.String("source", source), zap.Error(err)) + logger.Fatal("read source config", zap.String("source", source), zap.Error(err)) } version, err := loader.ReadVersion(sourceCfg) @@ -47,8 +47,7 @@ func run(source string) { logger.Fatal("read config version", zap.Error(err)) } if version != loader.V1 { - errStr := fmt.Sprintf("source config is not v%d, nothing to migrate", loader.V1) - logger.Fatal(errStr, zap.Int("version", version)) + logger.Fatal(fmt.Sprintf("source config is not v%d, nothing to migrate", loader.V1), zap.Int("version", version)) } backupPath := strings.TrimSuffix(source, filepath.Ext(source)) + ".bck" + filepath.Ext(source) @@ -56,14 +55,14 @@ func run(source string) { logger.Fatal("backup already exists, remove it before re-running", zap.String("backup", backupPath)) } if err := os.WriteFile(backupPath, sourceCfg, stat.Mode()); err != nil { - logger.Fatal("error writing backup legacy config", zap.String("backup", backupPath), zap.Error(err)) + logger.Fatal("write backup config", zap.String("backup", backupPath), zap.Error(err)) } var cfg v1.Config decoder := yaml.NewDecoder(bytes.NewReader(sourceCfg)) decoder.KnownFields(true) if err := decoder.Decode(&cfg); err != nil { - logger.Fatal("error parsing config v1", zap.Error(err)) + logger.Fatal("parse config v1", zap.Error(err)) } migratedCfg := migrate.V1ToV2(cfg) @@ -72,7 +71,7 @@ func run(source string) { encoder := yaml.NewEncoder(&buf) encoder.SetIndent(2) if err := encoder.Encode(&migratedCfg); err != nil { - logger.Fatal("error encoding config v2", zap.Error(err)) + logger.Fatal("encode config v2", zap.Error(err)) } if err := encoder.Close(); err != nil { logger.Fatal("close encoder", zap.Error(err)) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index bd7a3ac..3b811d7 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -142,7 +142,7 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac me := cfg.Handlers.MassExport redisCfg := cfg.Cache.RedisByID(me.SessionStore.RedisID) - exportServer, err := initExportService(ctx, *me, redisCfg, seqDBClients[cfg.Handlers.MassExport.SeqDBID]) + exportServer, err := initExportService(ctx, *me, redisCfg, seqDBClients[me.SeqDBID]) if err != nil { logger.Fatal("can't init export server", zap.Error(err)) } @@ -239,6 +239,7 @@ func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *con return seqdb.NewGRPCClient(ctx, clientParams) } + func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { if cfg == nil { logger.Warn("db config is nil, running without db") diff --git a/internal/app/config/loader/loader.go b/internal/app/config/loader/loader.go index 81401ad..6e81a58 100644 --- a/internal/app/config/loader/loader.go +++ b/internal/app/config/loader/loader.go @@ -25,29 +25,29 @@ type configMeta struct { func FromFile(cfgPath string) (v2.Config, error) { cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec if err != nil { - return v2.Config{}, fmt.Errorf("error reading file: %s", err) + return v2.Config{}, fmt.Errorf("error reading file: %w", err) } - meta, err := parse[configMeta](cfgBytes, false) + version, err := ReadVersion(cfgBytes) if err != nil { - return v2.Config{}, err + return v2.Config{}, fmt.Errorf("error reading version: %w", err) } cfg := v2.Config{} - switch { - case meta.Version == nil || *meta.Version == V1: + switch version { + case V1: cfgV1, err := parse[v1.Config](cfgBytes, true) if err != nil { - return v2.Config{}, err + return v2.Config{}, fmt.Errorf("error parsing config v1: %w", err) } cfg = migrate.V1ToV2(cfgV1) - case *meta.Version == V2: + case V2: cfg, err = parse[v2.Config](cfgBytes, true) if err != nil { - return v2.Config{}, err + return v2.Config{}, fmt.Errorf("error parsing config v2: %w", err) } default: - return v2.Config{}, fmt.Errorf("unsupported config version: %d", *meta.Version) + return v2.Config{}, fmt.Errorf("unsupported config version: %d", version) } if err := v2.Normalize(&cfg); err != nil { @@ -76,7 +76,7 @@ func parse[T any](cfg []byte, strict bool) (T, error) { decoder := yaml.NewDecoder(bytes.NewReader(cfg)) decoder.KnownFields(strict) if err := decoder.Decode(&result); err != nil { - return result, fmt.Errorf("error parsing config: %w", err) + return result, err } return result, nil diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index 2d4e73e..e177ffd 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -100,12 +100,7 @@ func migrateApiRateLimiters(src v1.ApiToRateLimiters) v2.ApiToRateLimiters { dst := make(v2.ApiToRateLimiters, len(src)) for api, rl := range src { dst[api] = v2.ApiRateLimiters{ - Default: v2.RateLimiter{ - RatePerSec: rl.Default.RatePerSec, - MaxBurst: rl.Default.MaxBurst, - StoreMaxKeys: rl.Default.StoreMaxKeys, - PerHandler: rl.Default.PerHandler, - }, + Default: migrateRateLimiter(rl.Default), SpecialUsers: migrateUserToRateLimiter(rl.SpecialUsers), } } @@ -120,17 +115,21 @@ func migrateUserToRateLimiter(src v1.UserToRateLimiter) v2.UserToRateLimiter { dst := make(v2.UserToRateLimiter, len(src)) for k, rl := range src { - dst[k] = v2.RateLimiter{ - RatePerSec: rl.RatePerSec, - MaxBurst: rl.MaxBurst, - StoreMaxKeys: rl.StoreMaxKeys, - PerHandler: rl.PerHandler, - } + dst[k] = migrateRateLimiter(rl) } return dst } +func migrateRateLimiter(src v1.RateLimiter) v2.RateLimiter { + return v2.RateLimiter{ + RatePerSec: src.RatePerSec, + MaxBurst: src.MaxBurst, + StoreMaxKeys: src.StoreMaxKeys, + PerHandler: src.PerHandler, + } +} + func migrateGRPCKeepaliveParams(src *v1.GRPCKeepaliveParams) *v2.GRPCKeepaliveParams { if src == nil { return nil @@ -256,6 +255,7 @@ func migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { MaxRetries: src.MaxRetries, MinRetryBackoff: src.MinRetryBackoff, MaxRetryBackoff: src.MaxRetryBackoff, + KeyPrefix: src.KeyPrefix, } } @@ -268,6 +268,7 @@ func migrateHandlers(src *v1.Handlers, cfg *v2.Config, v1Cache v1.Cache) *v2.Han SeqAPI: migrateSeqAPI(src.SeqAPI, v1Cache), ErrorGroups: migrateErrorGroups(src.ErrorGroups), AsyncSearch: migrateAsyncSearch(src.AsyncSearch), + Admin: migrateAdmin(src.Admin, v1Cache), } if src.MassExport != nil { @@ -496,6 +497,19 @@ func migrateDownloadParams(src *v1.SeqProxyDownloader) *v2.DownloadParams { } } +func migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { + if src == nil { + return nil + } + + _, redisID := defaultCacheIDs(v1Cache) + return &v2.Admin{ + RedisID: redisID, + SuperUsers: src.SuperUsers, + CacheTTL: src.CacheTTL, + } +} + func defaultCacheIDs(src v1.Cache) (cacheID, redisID string) { hasInmem := !isInmemEmpty(src.Inmemory) hasRedis := src.Redis != nil diff --git a/internal/app/config/v1/config.go b/internal/app/config/v1/config.go index b35be04..0d12a4e 100644 --- a/internal/app/config/v1/config.go +++ b/internal/app/config/v1/config.go @@ -57,6 +57,7 @@ import ( // max_retries: // min_retry_backoff: // max_retry_backoff: +// key_prefix: // db: // name: // host: diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 5745e21..c4baf42 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -85,7 +85,7 @@ import ( // seq_db_id: // cache_id: // redis_id: -// global_options: +// global_options: // events_cache_ttl: // pinned_fields: // name: @@ -176,6 +176,10 @@ import ( // default_env: // admin_users: // list_query_length_limit: +// admin: +// redis_id: +// super_users: +// cache_ttl: // db: // name: // host: @@ -201,6 +205,7 @@ import ( // max_retries: // min_retry_backoff: // max_retry_backoff: +// key_prefix: const ( DefaultSeqDBClientID = "defaultSeqDB" @@ -623,10 +628,10 @@ func Normalize(cfg *Config) error { for i := range cfg.Clients.SeqDB { c := &cfg.Clients.SeqDB[i] if c.ID == "" { - return fmt.Errorf("seq_db client ID cannot be empty") + return fmt.Errorf("clients.seq_db[%d].id cannot be empty", i) } if _, ok := seqDBIDs[c.ID]; ok { - return fmt.Errorf("duplicate seq_db client ID: %s", c.ID) + return fmt.Errorf("duplicate clients.seq_db.id %q", c.ID) } seqDBIDs[c.ID] = struct{}{} @@ -634,16 +639,12 @@ func Normalize(cfg *Config) error { if c.ClientMode == "" { c.ClientMode = ProxyClientModeGRPC } else if c.ClientMode != ProxyClientModeGRPC { - return fmt.Errorf("invalid clients.seq_db[%s].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, ProxyClientModeGRPC) + return fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, ProxyClientModeGRPC) } if c.GRPCKeepaliveParams != nil { - if c.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { - c.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime - } - if c.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { - c.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout - } + c.GRPCKeepaliveParams.Time = max(c.GRPCKeepaliveParams.Time, minGRPCKeepaliveTime) + c.GRPCKeepaliveParams.Timeout = max(c.GRPCKeepaliveParams.Timeout, minGRPCKeepaliveTimeout) } } @@ -651,10 +652,10 @@ func Normalize(cfg *Config) error { for i := range cfg.Clients.ClickHouse { ch := &cfg.Clients.ClickHouse[i] if ch.ID == "" { - return fmt.Errorf("clickhouse client ID cannot be empty") + return fmt.Errorf("clients.clickhouse[%d].id cannot be empty", i) } if _, ok := chIDs[ch.ID]; ok { - return fmt.Errorf("duplicate clickhouse client ID: %s", ch.ID) + return fmt.Errorf("duplicate clients.clickhouse.id %q", ch.ID) } chIDs[ch.ID] = struct{}{} @@ -671,10 +672,10 @@ func Normalize(cfg *Config) error { for i := range cfg.Cache.Inmemory { inm := &cfg.Cache.Inmemory[i] if inm.ID == "" { - return fmt.Errorf("inmemory cache ID cannot be empty") + return fmt.Errorf("cache.inmemory[%d].id cannot be empty", i) } if _, ok := inmemIDs[inm.ID]; ok { - return fmt.Errorf("duplicate inmemory cache ID: %s", inm.ID) + return fmt.Errorf("duplicate cache.inmemory.id %q", inm.ID) } inmemIDs[inm.ID] = struct{}{} @@ -694,17 +695,17 @@ func Normalize(cfg *Config) error { for i := range cfg.Cache.Redis { r := &cfg.Cache.Redis[i] if r.ID == "" { - return fmt.Errorf("redis cache ID cannot be empty") + return fmt.Errorf("cache.redis[%d].id cannot be empty", i) } if _, ok := redisIDs[r.ID]; ok { - return fmt.Errorf("duplicate redis cache ID: %s", r.ID) + return fmt.Errorf("duplicate cache.redis.id %q", r.ID) } redisIDs[r.ID] = struct{}{} if r.WithInmemID != "" { if _, ok := inmemIDs[r.WithInmemID]; !ok { - return fmt.Errorf("redis cache %q references unknown inmem cache id %q", r.ID, r.WithInmemID) + return fmt.Errorf("cache.redis[%q].with_inmem_id %q not found in cache.inmemory", r.ID, r.WithInmemID) } } } @@ -723,31 +724,36 @@ func Normalize(cfg *Config) error { if len(cfg.Handlers.SeqAPI.Envs) > 0 { if cfg.Handlers.SeqAPI.SeqDBID != "" { - return fmt.Errorf("seq_api.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + return fmt.Errorf("handlers.seq_api.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") } - if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return fmt.Errorf("default_env must be specified when using envs") + return fmt.Errorf("handlers.seq_api.default_env must be specified when envs is used") } - if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) + return fmt.Errorf("handlers.seq_api.default_env %q not found in envs", cfg.Handlers.SeqAPI.DefaultEnv) } for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { + if envConfig.SeqDBID == "" { + return fmt.Errorf("handlers.seq_api.envs[%q].seq_db_id cannot be empty", envName) + } if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { - return fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDBID, envName) + return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + } + + if envConfig.Options == nil { + envConfig.Options = &cfg.Handlers.SeqAPI.Options + } else { + merged := mergeSeqAPIOptions(cfg.Handlers.SeqAPI.Options, *envConfig.Options) + envConfig.Options = &merged } - merged := mergeSeqAPIOptions(cfg.Handlers.SeqAPI.Options, envConfig.Options) - envConfig.Options = &merged cfg.Handlers.SeqAPI.Envs[envName] = envConfig } } else { if cfg.Handlers.SeqAPI.SeqDBID == "" { - return fmt.Errorf("seq_api.seq_db_id must be specified when envs is empty") + return fmt.Errorf("handlers.seq_api.seq_db_id cannot be empty when envs is not used") } - if _, ok := seqDBIDs[cfg.Handlers.SeqAPI.SeqDBID]; !ok { return fmt.Errorf("unknown handlers.seq_api.seq_db_id %q", cfg.Handlers.SeqAPI.SeqDBID) } @@ -773,29 +779,31 @@ func Normalize(cfg *Config) error { if len(cfg.Handlers.AsyncSearch.Envs) > 0 { if cfg.Handlers.AsyncSearch.SeqDBID != "" { - return fmt.Errorf("async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + return fmt.Errorf("handlers.async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") } - if cfg.Handlers.AsyncSearch.DefaultEnv == "" { - return fmt.Errorf("default_env must be specified when using envs") + return fmt.Errorf("handlers.async_search.default_env must be specified when using envs") } - - if _, exists := cfg.Handlers.AsyncSearch.Envs[cfg.Handlers.AsyncSearch.DefaultEnv]; !exists { - return fmt.Errorf("default_env '%s' not found in async_search.envs", cfg.Handlers.AsyncSearch.DefaultEnv) + if _, ok := cfg.Handlers.AsyncSearch.Envs[cfg.Handlers.AsyncSearch.DefaultEnv]; !ok { + return fmt.Errorf("handlers.async_search.default_env %q not found in envs", cfg.Handlers.AsyncSearch.DefaultEnv) } for envName, envConfig := range cfg.Handlers.AsyncSearch.Envs { if envConfig.SeqDBID == "" { return fmt.Errorf("handlers.async_search.envs[%q].seq_db_id cannot be empty", envName) } - - if _, ok := seqDBIDs[cfg.Handlers.AsyncSearch.SeqDBID]; !ok { + if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) } + if envConfig.ListQueryLengthLimit <= 0 { + envConfig.ListQueryLengthLimit = cfg.Handlers.AsyncSearch.ListQueryLengthLimit + } + + cfg.Handlers.AsyncSearch.Envs[envName] = envConfig } } else { if cfg.Handlers.AsyncSearch.SeqDBID == "" { - return fmt.Errorf("handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) + return fmt.Errorf("handlers.async_search.seq_db_id cannot be empty when envs is not used") } if _, ok := seqDBIDs[cfg.Handlers.AsyncSearch.SeqDBID]; !ok { return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) @@ -808,29 +816,25 @@ func Normalize(cfg *Config) error { if eg.CHID != "" { return fmt.Errorf("handlers.error_groups.ch_id must be empty when envs is used. Put ch_id inside each env") } - if eg.DefaultEnv == "" { - return fmt.Errorf("default_env must be specified when using envs") + return fmt.Errorf("handlers.error_groups.default_env must be specified when using envs") } - - if _, exists := eg.Envs[eg.DefaultEnv]; !exists { - return fmt.Errorf("default_env '%s' not found in error_groups.envs", eg.DefaultEnv) + if _, ok := eg.Envs[eg.DefaultEnv]; !ok { + return fmt.Errorf("handlers.error_groups.default_env %q not found in envs", eg.DefaultEnv) } for envName, envConfig := range eg.Envs { if envConfig.CHID == "" { - return fmt.Errorf("client '%s' for env '%s' not found", envConfig.CHID, envName) + return fmt.Errorf("handlers.error_groups.envs[%q].ch_id cannot be empty", envName) } - if _, ok := chIDs[envConfig.CHID]; !ok { - return fmt.Errorf("unknown handlers.error_groups.ch_id %q", envConfig.CHID) + return fmt.Errorf("unknown handlers.error_groups.envs[%q].ch_id %q", envName, envConfig.CHID) } } } else { if cfg.Handlers.ErrorGroups.CHID == "" { - return fmt.Errorf("handlers.error_groups.ch_id cannot be empty") + return fmt.Errorf("handlers.error_groups.ch_id cannot be empty when envs is not used") } - if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) } @@ -894,11 +898,8 @@ func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { } } -func mergeSeqAPIOptions(global SeqAPIOptions, envOptions *SeqAPIOptions) SeqAPIOptions { +func mergeSeqAPIOptions(global SeqAPIOptions, envOptions SeqAPIOptions) SeqAPIOptions { merged := global - if envOptions == nil { - return merged - } if envOptions.MaxAggregationsPerRequest > 0 { merged.MaxAggregationsPerRequest = envOptions.MaxAggregationsPerRequest diff --git a/internal/app/server/init.go b/internal/app/server/init.go index b3545a8..cf49e75 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -46,7 +46,7 @@ func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { err = s.prepareDebugServer(ctx) if err != nil { - return nil + return err } return nil diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index df88796..2979195 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -60,8 +60,8 @@ func New(ctx context.Context, inmemCfg *config.InmemoryCache, redisCfg *config.R } if redisErr != nil { - logger.Warn("failed to init redis cache; inmemory cache will be used instead", zap.Error(err)) - return inmem, err + logger.Warn("failed to init redis cache; inmemory cache will be used instead", zap.Error(redisErr)) + return inmem, redisErr } return &inmemWithRedis{ From 1e31dd1f6cd0b6d3fdf436d47e0c192c6aaed066 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 2 Sep 2026 00:00:47 +0300 Subject: [PATCH 12/29] update config package(merge, loader, config scheme) --- cmd/config-migrate/main.go | 41 +--- cmd/seq-ui/main.go | 2 +- .../api/seqapi/v1/grpc/aggregation_test.go | 2 +- internal/api/seqapi/v1/grpc/api.go | 2 +- internal/api/seqapi/v1/grpc/events_test.go | 2 +- internal/api/seqapi/v1/grpc/fields_test.go | 2 +- internal/api/seqapi/v1/grpc/get_envs_test.go | 2 +- internal/api/seqapi/v1/grpc/limits_test.go | 2 +- .../api/seqapi/v1/grpc/logs_lifespan_test.go | 2 +- internal/api/seqapi/v1/grpc/search_test.go | 2 +- internal/api/seqapi/v1/grpc/test_data.go | 2 +- .../api/seqapi/v1/http/aggregation_test.go | 2 +- .../api/seqapi/v1/http/aggregation_ts_test.go | 2 +- internal/api/seqapi/v1/http/api.go | 2 +- internal/api/seqapi/v1/http/events_test.go | 2 +- .../v1/http/export_async_search_test.go | 2 +- internal/api/seqapi/v1/http/export_test.go | 2 +- internal/api/seqapi/v1/http/fields_test.go | 2 +- internal/api/seqapi/v1/http/get_envs_test.go | 2 +- internal/api/seqapi/v1/http/limits_test.go | 2 +- .../api/seqapi/v1/http/logs_lifespan_test.go | 2 +- internal/api/seqapi/v1/http/search_test.go | 2 +- internal/api/seqapi/v1/http/test_data.go | 2 +- internal/api/seqapi/v1/seqapi.go | 2 +- internal/api/seqapi/v1/test/data.go | 2 +- internal/app/auth/oidc.go | 2 +- internal/app/config/loader/loader.go | 129 ++++++++++++- internal/app/config/v1/config.go | 94 ++++----- internal/app/config/v2/config.go | 182 +++++++----------- internal/app/mw/auth.go | 2 +- internal/app/mw/ratelimiter.go | 2 +- internal/app/ratelimiter/ratelimiter.go | 2 +- internal/app/server/init.go | 12 +- internal/app/server/server.go | 2 +- internal/pkg/cache/ctor.go | 2 +- internal/pkg/cache/inmemory.go | 2 +- internal/pkg/cache/redis.go | 2 +- internal/pkg/mask/field_filter.go | 2 +- internal/pkg/mask/field_filter_test.go | 2 +- internal/pkg/mask/mask.go | 2 +- internal/pkg/mask/mask_test.go | 2 +- internal/pkg/mask/masker.go | 2 +- internal/pkg/mask/masker_test.go | 2 +- internal/pkg/redisclient/client.go | 2 +- internal/pkg/service/admin/roles_test.go | 2 +- internal/pkg/service/admin/service.go | 2 +- .../pkg/service/async_searches/service.go | 2 +- internal/pkg/service/errorgroups/service.go | 2 +- .../pkg/service/errorgroups/service_test.go | 2 +- .../pkg/service/massexport/filestore/s3.go | 2 +- .../massexport/seq_proxy_downloader.go | 2 +- internal/pkg/service/massexport/service.go | 2 +- .../service/massexport/sessionstore/redis.go | 2 +- 53 files changed, 307 insertions(+), 247 deletions(-) diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go index 97583e7..f8f99e5 100644 --- a/cmd/config-migrate/main.go +++ b/cmd/config-migrate/main.go @@ -1,19 +1,15 @@ package main import ( - "bytes" + "errors" "flag" - "fmt" "os" "path/filepath" "strings" "go.uber.org/zap" - "gopkg.in/yaml.v3" "github.com/ozontech/seq-ui/internal/app/config/loader" - "github.com/ozontech/seq-ui/internal/app/config/migrate" - v1 "github.com/ozontech/seq-ui/internal/app/config/v1" "github.com/ozontech/seq-ui/logger" ) @@ -29,7 +25,7 @@ func main() { func run(source string) { if source == "" { - logger.Fatal("missing required parameter", zap.String("param", "-source")) + logger.Fatal("miss required parameter", zap.String("param", "-source")) } stat, err := os.Stat(source) @@ -42,12 +38,12 @@ func run(source string) { logger.Fatal("read source config", zap.String("source", source), zap.Error(err)) } - version, err := loader.ReadVersion(sourceCfg) - if err != nil { - logger.Fatal("read config version", zap.Error(err)) - } - if version != loader.V1 { - logger.Fatal(fmt.Sprintf("source config is not v%d, nothing to migrate", loader.V1), zap.Int("version", version)) + migratedCfg, err := loader.ToLatestVersion(sourceCfg) + if errors.Is(err, loader.ErrAlreadyLatestConfigVersion) { + logger.Info("source config is already at the latest schema version, nothing to migrate", zap.String("source", source)) + return + } else if err != nil { + logger.Fatal("migrate config", zap.String("source", source), zap.Error(err)) } backupPath := strings.TrimSuffix(source, filepath.Ext(source)) + ".bck" + filepath.Ext(source) @@ -58,26 +54,7 @@ func run(source string) { logger.Fatal("write backup config", zap.String("backup", backupPath), zap.Error(err)) } - var cfg v1.Config - decoder := yaml.NewDecoder(bytes.NewReader(sourceCfg)) - decoder.KnownFields(true) - if err := decoder.Decode(&cfg); err != nil { - logger.Fatal("parse config v1", zap.Error(err)) - } - - migratedCfg := migrate.V1ToV2(cfg) - - var buf bytes.Buffer - encoder := yaml.NewEncoder(&buf) - encoder.SetIndent(2) - if err := encoder.Encode(&migratedCfg); err != nil { - logger.Fatal("encode config v2", zap.Error(err)) - } - if err := encoder.Close(); err != nil { - logger.Fatal("close encoder", zap.Error(err)) - } - - if err := os.WriteFile(source, buf.Bytes(), stat.Mode()); err != nil { + if err := os.WriteFile(source, migratedCfg, stat.Mode()); err != nil { logger.Fatal("write migrated config", zap.Error(err)) } diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 3b811d7..e1b24ad 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -25,7 +25,7 @@ import ( seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" "github.com/ozontech/seq-ui/internal/app/config/loader" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/server" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" diff --git a/internal/api/seqapi/v1/grpc/aggregation_test.go b/internal/api/seqapi/v1/grpc/aggregation_test.go index 7ee158b..bc1261a 100644 --- a/internal/api/seqapi/v1/grpc/aggregation_test.go +++ b/internal/api/seqapi/v1/grpc/aggregation_test.go @@ -12,7 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index dcdb7e8..d425633 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -10,7 +10,7 @@ import ( "go.uber.org/zap" "google.golang.org/grpc/metadata" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" diff --git a/internal/api/seqapi/v1/grpc/events_test.go b/internal/api/seqapi/v1/grpc/events_test.go index b910470..736b6c8 100644 --- a/internal/api/seqapi/v1/grpc/events_test.go +++ b/internal/api/seqapi/v1/grpc/events_test.go @@ -14,7 +14,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" diff --git a/internal/api/seqapi/v1/grpc/fields_test.go b/internal/api/seqapi/v1/grpc/fields_test.go index 5df65b0..0e78bc0 100644 --- a/internal/api/seqapi/v1/grpc/fields_test.go +++ b/internal/api/seqapi/v1/grpc/fields_test.go @@ -11,7 +11,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/grpc/get_envs_test.go b/internal/api/seqapi/v1/grpc/get_envs_test.go index 4deb8be..0acda54 100644 --- a/internal/api/seqapi/v1/grpc/get_envs_test.go +++ b/internal/api/seqapi/v1/grpc/get_envs_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/grpc/limits_test.go b/internal/api/seqapi/v1/grpc/limits_test.go index f38efe3..3d1bc4f 100644 --- a/internal/api/seqapi/v1/grpc/limits_test.go +++ b/internal/api/seqapi/v1/grpc/limits_test.go @@ -8,7 +8,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index cf4ddff..18095e7 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -13,7 +13,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" diff --git a/internal/api/seqapi/v1/grpc/search_test.go b/internal/api/seqapi/v1/grpc/search_test.go index ce1644c..148b59e 100644 --- a/internal/api/seqapi/v1/grpc/search_test.go +++ b/internal/api/seqapi/v1/grpc/search_test.go @@ -12,7 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index b7464e6..ff0d6ab 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -5,7 +5,7 @@ import ( "time" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) diff --git a/internal/api/seqapi/v1/http/aggregation_test.go b/internal/api/seqapi/v1/http/aggregation_test.go index 6b75c75..76978d9 100644 --- a/internal/api/seqapi/v1/http/aggregation_test.go +++ b/internal/api/seqapi/v1/http/aggregation_test.go @@ -11,7 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/http/aggregation_ts_test.go b/internal/api/seqapi/v1/http/aggregation_ts_test.go index cd06b34..d6a8752 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts_test.go +++ b/internal/api/seqapi/v1/http/aggregation_ts_test.go @@ -17,7 +17,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index a66e32d..09d0133 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -10,7 +10,7 @@ import ( "github.com/gofrs/uuid" "go.uber.org/zap" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/tokenlimiter" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index 606bf3c..bed5b93 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -14,7 +14,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" diff --git a/internal/api/seqapi/v1/http/export_async_search_test.go b/internal/api/seqapi/v1/http/export_async_search_test.go index a5edc1c..b640fee 100644 --- a/internal/api/seqapi/v1/http/export_async_search_test.go +++ b/internal/api/seqapi/v1/http/export_async_search_test.go @@ -8,7 +8,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/http/export_test.go b/internal/api/seqapi/v1/http/export_test.go index 5c36e14..2bedd30 100644 --- a/internal/api/seqapi/v1/http/export_test.go +++ b/internal/api/seqapi/v1/http/export_test.go @@ -10,7 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index 3d6a26b..7bbd102 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -9,7 +9,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/http/get_envs_test.go b/internal/api/seqapi/v1/http/get_envs_test.go index 53ddae3..6e46ced 100644 --- a/internal/api/seqapi/v1/http/get_envs_test.go +++ b/internal/api/seqapi/v1/http/get_envs_test.go @@ -6,7 +6,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestServeGetEnvs(t *testing.T) { diff --git a/internal/api/seqapi/v1/http/limits_test.go b/internal/api/seqapi/v1/http/limits_test.go index 1a8dca9..eebb094 100644 --- a/internal/api/seqapi/v1/http/limits_test.go +++ b/internal/api/seqapi/v1/http/limits_test.go @@ -6,7 +6,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestServeGetLimits(t *testing.T) { diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index bd36447..3238563 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -12,7 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" diff --git a/internal/api/seqapi/v1/http/search_test.go b/internal/api/seqapi/v1/http/search_test.go index 9e92656..945674f 100644 --- a/internal/api/seqapi/v1/http/search_test.go +++ b/internal/api/seqapi/v1/http/search_test.go @@ -11,7 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index efab1f0..6e0c0ff 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -9,7 +9,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index bd04cb3..375e0e2 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -5,7 +5,7 @@ import ( grpc_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/grpc" http_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/http" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" diff --git a/internal/api/seqapi/v1/test/data.go b/internal/api/seqapi/v1/test/data.go index b32962f..9af766c 100644 --- a/internal/api/seqapi/v1/test/data.go +++ b/internal/api/seqapi/v1/test/data.go @@ -6,7 +6,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" mock_asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches/mock" diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 0dbfb24..790c2b8 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -13,7 +13,7 @@ import ( "github.com/coreos/go-oidc/v3/oidc" "go.uber.org/zap" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/tls" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" diff --git a/internal/app/config/loader/loader.go b/internal/app/config/loader/loader.go index 6e81a58..8cf715d 100644 --- a/internal/app/config/loader/loader.go +++ b/internal/app/config/loader/loader.go @@ -2,7 +2,9 @@ package loader import ( "bytes" + "errors" "fmt" + "maps" "os" "gopkg.in/yaml.v3" @@ -17,6 +19,8 @@ const ( V2 = 2 ) +var ErrAlreadyLatestConfigVersion = errors.New("config is already at the latest schema version") + type configMeta struct { Version *int `yaml:"version"` } @@ -25,31 +29,39 @@ type configMeta struct { func FromFile(cfgPath string) (v2.Config, error) { cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec if err != nil { - return v2.Config{}, fmt.Errorf("error reading file: %w", err) + return v2.Config{}, fmt.Errorf("read file: %w", err) } version, err := ReadVersion(cfgBytes) if err != nil { - return v2.Config{}, fmt.Errorf("error reading version: %w", err) + return v2.Config{}, fmt.Errorf("read version: %w", err) } - cfg := v2.Config{} switch version { case V1: cfgV1, err := parse[v1.Config](cfgBytes, true) if err != nil { - return v2.Config{}, fmt.Errorf("error parsing config v1: %w", err) + return v2.Config{}, fmt.Errorf("parse config v1: %w", err) } - cfg = migrate.V1ToV2(cfgV1) - case V2: - cfg, err = parse[v2.Config](cfgBytes, true) + cfgBytes, err = encode(migrate.V1ToV2(cfgV1)) if err != nil { - return v2.Config{}, fmt.Errorf("error parsing config v2: %w", err) + return v2.Config{}, nil } + case V2: default: return v2.Config{}, fmt.Errorf("unsupported config version: %d", version) } + cfgBytes, err = mergeHandlersEnvOptions(cfgBytes) + if err != nil { + return v2.Config{}, fmt.Errorf("merge env options: %w", err) + } + + cfg, err := parse[v2.Config](cfgBytes, true) + if err != nil { + return v2.Config{}, fmt.Errorf("error parsing config v2: %w", err) + } + if err := v2.Normalize(&cfg); err != nil { return v2.Config{}, fmt.Errorf("normalize config: %w", err) } @@ -57,6 +69,26 @@ func FromFile(cfgPath string) (v2.Config, error) { return cfg, nil } +func ToLatestVersion(cfgBytes []byte) ([]byte, error) { + version, err := ReadVersion(cfgBytes) + if err != nil { + return nil, fmt.Errorf("read version: %w", err) + } + + switch version { + case V1: + cfgV1, err := parse[v1.Config](cfgBytes, true) + if err != nil { + return nil, fmt.Errorf("parse config v1: %w", err) + } + return encode(migrate.V1ToV2(cfgV1)) + case V2: + return nil, ErrAlreadyLatestConfigVersion + default: + return nil, fmt.Errorf("unsupported config version: %d", version) + } +} + func ReadVersion(cfgBytes []byte) (int, error) { meta, err := parse[configMeta](cfgBytes, false) if err != nil { @@ -81,3 +113,84 @@ func parse[T any](cfg []byte, strict bool) (T, error) { return result, nil } + +func encode(cfg v2.Config) ([]byte, error) { + var buf bytes.Buffer + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + if err := encoder.Encode(&cfg); err != nil { + return nil, fmt.Errorf("encode config v2: %w", err) + } + if err := encoder.Close(); err != nil { + return nil, fmt.Errorf("close encoder: %w", err) + } + return buf.Bytes(), nil +} + +func mergeHandlersEnvOptions(cfgBytes []byte) ([]byte, error) { + var root map[any]any + if err := yaml.Unmarshal(cfgBytes, &root); err != nil { + return nil, fmt.Errorf("parse yaml for merge: %w", err) + } + + handlers := getMap(root, "handlers") + for _, h := range handlers { + handler, _ := h.(map[any]any) + if handler == nil { + continue + } + + rootOpts := getMap(handler, "options") + envs := getMap(handler, "envs") + if rootOpts == nil || len(envs) == 0 { + continue + } + + for _, e := range envs { + env, _ := e.(map[any]any) + if env == nil { + continue + } + + envOpts, _ := env["options"].(map[any]any) + env["options"] = mergeYAMLs(rootOpts, envOpts) + } + } + + out, err := yaml.Marshal(root) + if err != nil { + return nil, fmt.Errorf("marshal merged yaml: %w", err) + } + return out, nil +} + +func getMap(root map[any]any, path ...string) map[any]any { + cur := root + for _, p := range path { + next, _ := cur[p].(map[any]any) + if next == nil { + return nil + } + cur = next + } + + return cur +} + +func mergeYAMLs(a, b map[any]any) map[any]any { + merged := make(map[any]any) + maps.Copy(merged, a) + + for k, v := range b { + if existingValue, exists := merged[k]; exists { + if existingMap, ok := existingValue.(map[any]any); ok { + if newMap, ok := v.(map[any]any); ok { + merged[k] = mergeYAMLs(existingMap, newMap) + continue + } + } + } + merged[k] = v + } + return merged +} diff --git a/internal/app/config/v1/config.go b/internal/app/config/v1/config.go index 0d12a4e..b1b34c8 100644 --- a/internal/app/config/v1/config.go +++ b/internal/app/config/v1/config.go @@ -1,10 +1,10 @@ -package v1 +package config import ( "time" ) -// Deprecated configuration scheme +// Configuration scheme // server: // http_addr: // grpc_addr: @@ -50,6 +50,7 @@ import ( // max_cost: // buffer_items: // redis: +// key_prefix: // addr: // username: // password: @@ -57,7 +58,6 @@ import ( // max_retries: // min_retry_backoff: // max_retry_backoff: -// key_prefix: // db: // name: // host: @@ -88,18 +88,18 @@ import ( // timeout: // permit_without_stream: // seq_db: -// id: -// timeout: -// avg_doc_size: -// addrs: -// request_retries: -// initial_retry_backoff: -// max_retry_backoff: -// client_mode: -// grpc_keepalive_params: -// time: +// - id: // timeout: -// permit_without_stream: +// avg_doc_size: +// addrs: +// request_retries: +// initial_retry_backoff: +// max_retry_backoff: +// client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: // handlers: // seq_api: // max_search_limit: @@ -112,30 +112,30 @@ import ( // max_buckets_per_aggregation_ts: // events_cache_ttl: // pinned_fields: -// name: -// type: +// - name: +// type: // system_fields: -// name: -// type: +// - name: +// type: // logs_lifespan_cache_key: // logs_lifespan_cache_ttl: // fields_cache_ttl: // masking: // masks: -// re: -// groups: -// mode: -// replace_word: +// - re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// - field: +// mode: +// values: // process_fields: // ignore_fields: -// field_filters: -// condition: -// filters: -// field: -// mode: -// values: -// process_fields: -// ignore_fields: // envs: // : // seq_db_id: @@ -150,28 +150,28 @@ import ( // max_buckets_per_aggregation_ts: // events_cache_ttl: // pinned_fields: -// name: -// type: +// - name: +// type: // system_fields: -// name: -// type: +// - name: +// type: // logs_lifespan_cache_key: // logs_lifespan_cache_ttl: // fields_cache_ttl: // masking: // masks: -// re: -// groups: -// mode: -// replace_word: -// process_fields: -// ignore_fields: -// field_filters: -// condition: -// filters: -// field: -// mode: -// values: +// - re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// - field: +// mode: +// values: // process_fields: // ignore_fields: // default_env: @@ -288,6 +288,7 @@ type InmemoryCache struct { } type Redis struct { + KeyPrefix string `yaml:"key_prefix"` Addr string `yaml:"addr"` Username string `yaml:"username"` Password string `yaml:"password"` @@ -295,7 +296,6 @@ type Redis struct { MaxRetries int `yaml:"max_retries"` MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - KeyPrefix string `yaml:"key_prefix"` } type Cache struct { diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index c4baf42..f02060f 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -1,11 +1,11 @@ -package v2 +package config import ( "fmt" "time" ) -// Actual configuration scheme +// Configuration scheme // version: // server: // http: @@ -55,31 +55,31 @@ import ( // per_handler: // clients: // seq_db: -// id: -// timeout: -// avg_doc_size: -// addrs: -// request_retries: -// initial_retry_backoff: -// max_retry_backoff: -// client_mode: -// grpc_keepalive_params: -// time: +// - id: // timeout: -// permit_without_stream: -// download_params: -// delay: +// avg_doc_size: +// addrs: +// request_retries: // initial_retry_backoff: // max_retry_backoff: +// client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: +// download_params: +// delay: +// initial_retry_backoff: +// max_retry_backoff: // clickhouse: -// id: -// addrs: -// database: -// username: -// password: -// sharded: -// dial_timeout: -// read_timeout: +// - id: +// addrs: +// database: +// username: +// password: +// sharded: +// dial_timeout: +// read_timeout: // handlers: // seq_api: // seq_db_id: @@ -88,27 +88,27 @@ import ( // global_options: // events_cache_ttl: // pinned_fields: -// name: -// type: +// - name: +// type: // system_fields: -// name: -// type: +// - name: +// type: // logs_lifespan_cache_ttl: // fields_cache_ttl: // masking: // masks: -// re: -// groups: -// mode: -// replace_word: -// process_fields: -// ignore_fields: -// field_filters: -// condition: -// filters: -// field: -// mode: -// values: +// - re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// - field: +// mode: +// values: // process_fields: // ignore_fields: // options: @@ -191,31 +191,31 @@ import ( // use_prepared_statements: // cache: // inmemory: -// id: -// num_counters: -// max_cost: -// buffer_items: +// - id: +// num_counters: +// max_cost: +// buffer_items: // redis: -// id: -// with_inmem_id: -// addr: -// username: -// password: -// timeout: -// max_retries: -// min_retry_backoff: -// max_retry_backoff: -// key_prefix: +// - id: +// key_prefix: +// with_inmem_id: +// addr: +// username: +// password: +// timeout: +// max_retries: +// min_retry_backoff: +// max_retry_backoff: const ( - DefaultSeqDBClientID = "defaultSeqDB" - DefaultCHClientID = "defaultCH" - DefaultInmemCacheID = "defaultInmemCache" - DefaultRedisID = "defaultRedis" - DefaultRedis2ID = "defaultRedis2" + DefaultSeqDBClientID = "default_seq_db" + DefaultCHClientID = "default_ch" + DefaultInmemCacheID = "default_inmem_cache" + DefaultRedisID = "default_redis" + DefaultRedis2ID = "default_redis_2" DefaultMassExportRedisID = "mass_export" - ProxyClientModeGRPC = "grpc" + SeqDBClientModeGRPC = "grpc" MaskModeMask = "mask" MaskModeReplace = "replace" @@ -333,6 +333,7 @@ type InmemoryCache struct { type Redis struct { ID string `yaml:"id"` + KeyPrefix string `yaml:"key_prefix"` WithInmemID string `yaml:"with_inmem_id"` Addr string `yaml:"addr"` Username string `yaml:"username"` @@ -341,7 +342,6 @@ type Redis struct { MaxRetries int `yaml:"max_retries"` MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - KeyPrefix string `yaml:"key_prefix"` } type Cache struct { @@ -549,14 +549,14 @@ type SeqAPIEnv struct { } type SeqAPIOptions struct { - MaxSearchLimit int32 `yaml:"max_search_limit"` - MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` - MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` - MaxExportLimit int32 `yaml:"max_export_limit"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` + MaxSearchLimit int32 `yaml:"max_search_limit,omitempty"` + MaxSearchTotalLimit int64 `yaml:"max_search_total_limit,omitempty"` + MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit,omitempty"` + MaxExportLimit int32 `yaml:"max_export_limit,omitempty"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit,omitempty"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests,omitempty"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request,omitempty"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts,omitempty"` } type Masking struct { @@ -637,9 +637,9 @@ func Normalize(cfg *Config) error { seqDBIDs[c.ID] = struct{}{} if c.ClientMode == "" { - c.ClientMode = ProxyClientModeGRPC - } else if c.ClientMode != ProxyClientModeGRPC { - return fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, ProxyClientModeGRPC) + c.ClientMode = SeqDBClientModeGRPC + } else if c.ClientMode != SeqDBClientModeGRPC { + return fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, SeqDBClientModeGRPC) } if c.GRPCKeepaliveParams != nil { @@ -740,15 +740,6 @@ func Normalize(cfg *Config) error { if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) } - - if envConfig.Options == nil { - envConfig.Options = &cfg.Handlers.SeqAPI.Options - } else { - merged := mergeSeqAPIOptions(cfg.Handlers.SeqAPI.Options, *envConfig.Options) - envConfig.Options = &merged - } - - cfg.Handlers.SeqAPI.Envs[envName] = envConfig } } else { if cfg.Handlers.SeqAPI.SeqDBID == "" { @@ -898,37 +889,6 @@ func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { } } -func mergeSeqAPIOptions(global SeqAPIOptions, envOptions SeqAPIOptions) SeqAPIOptions { - merged := global - - if envOptions.MaxAggregationsPerRequest > 0 { - merged.MaxAggregationsPerRequest = envOptions.MaxAggregationsPerRequest - } - if envOptions.MaxBucketsPerAggregationTs > 0 { - merged.MaxBucketsPerAggregationTs = envOptions.MaxBucketsPerAggregationTs - } - if envOptions.MaxParallelExportRequests > 0 { - merged.MaxParallelExportRequests = envOptions.MaxParallelExportRequests - } - if envOptions.MaxSearchLimit > 0 { - merged.MaxSearchLimit = envOptions.MaxSearchLimit - } - if envOptions.MaxSearchTotalLimit > 0 { - merged.MaxSearchTotalLimit = envOptions.MaxSearchTotalLimit - } - if envOptions.MaxSearchOffsetLimit > 0 { - merged.MaxSearchOffsetLimit = envOptions.MaxSearchOffsetLimit - } - if envOptions.MaxExportLimit > 0 { - merged.MaxExportLimit = envOptions.MaxExportLimit - } - if envOptions.SeqCLIMaxSearchLimit > 0 { - merged.SeqCLIMaxSearchLimit = envOptions.SeqCLIMaxSearchLimit - } - - return merged -} - func setSeqAPIGlobalOptionsDefaults(options *SeqAPIGlobalOptions) { if options.EventsCacheTTL <= 0 { options.EventsCacheTTL = defaultEventsCacheTTL diff --git a/internal/app/mw/auth.go b/internal/app/mw/auth.go index cf7c2ad..4c3c486 100644 --- a/internal/app/mw/auth.go +++ b/internal/app/mw/auth.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/ozontech/seq-ui/internal/app/auth" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" ) diff --git a/internal/app/mw/ratelimiter.go b/internal/app/mw/ratelimiter.go index ccee177..07cf642 100644 --- a/internal/app/mw/ratelimiter.go +++ b/internal/app/mw/ratelimiter.go @@ -7,7 +7,7 @@ import ( "net/http" "strconv" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/ratelimiter" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/metric" diff --git a/internal/app/ratelimiter/ratelimiter.go b/internal/app/ratelimiter/ratelimiter.go index 096ac91..47ed459 100644 --- a/internal/app/ratelimiter/ratelimiter.go +++ b/internal/app/ratelimiter/ratelimiter.go @@ -8,7 +8,7 @@ import ( "github.com/throttled/throttled/v2" "github.com/throttled/throttled/v2/store/memstore" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) const rateLimiterQuantityPerCall = 1 diff --git a/internal/app/server/init.go b/internal/app/server/init.go index cf49e75..ab760da 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -15,6 +15,7 @@ import ( "google.golang.org/grpc/reflection" "github.com/ozontech/seq-ui/internal/api" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/mw" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/tracing" @@ -29,8 +30,17 @@ var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { var err error + var jwtSecretKey string + var oidcCfg *config.OIDC - s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.Auth.JWT.SecretKey, s.config.Auth.OIDC, s.oidcCache) + if s.config.Auth != nil { + if s.config.Auth.JWT != nil { + jwtSecretKey = s.config.Auth.JWT.SecretKey + } + oidcCfg = s.config.Auth.OIDC + } + + s.authPrvds, err = mw.NewAuthProviders(ctx, jwtSecretKey, oidcCfg, s.oidcCache) if err != nil { return err } diff --git a/internal/app/server/server.go b/internal/app/server/server.go index 1b5a0ca..81afc0a 100644 --- a/internal/app/server/server.go +++ b/internal/app/server/server.go @@ -8,7 +8,7 @@ import ( "google.golang.org/grpc" "github.com/ozontech/seq-ui/internal/api" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/mw" "github.com/ozontech/seq-ui/internal/pkg/cache" ) diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index 2979195..e8b5a50 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -6,7 +6,7 @@ import ( "go.uber.org/zap" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/logger" ) diff --git a/internal/pkg/cache/inmemory.go b/internal/pkg/cache/inmemory.go index 1236b87..d0e472e 100644 --- a/internal/pkg/cache/inmemory.go +++ b/internal/pkg/cache/inmemory.go @@ -7,7 +7,7 @@ import ( "github.com/dgraph-io/ristretto" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/metric" ) diff --git a/internal/pkg/cache/redis.go b/internal/pkg/cache/redis.go index 9f9dc7d..6eff755 100644 --- a/internal/pkg/cache/redis.go +++ b/internal/pkg/cache/redis.go @@ -8,7 +8,7 @@ import ( "github.com/redis/go-redis/v9" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/redisclient" "github.com/ozontech/seq-ui/metric" ) diff --git a/internal/pkg/mask/field_filter.go b/internal/pkg/mask/field_filter.go index 3f108ba..156077c 100644 --- a/internal/pkg/mask/field_filter.go +++ b/internal/pkg/mask/field_filter.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) type filterCondition int8 diff --git a/internal/pkg/mask/field_filter_test.go b/internal/pkg/mask/field_filter_test.go index fe69ba7..885dec4 100644 --- a/internal/pkg/mask/field_filter_test.go +++ b/internal/pkg/mask/field_filter_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestFieldFilterSetCreate(t *testing.T) { diff --git a/internal/pkg/mask/mask.go b/internal/pkg/mask/mask.go index c16035f..523c096 100644 --- a/internal/pkg/mask/mask.go +++ b/internal/pkg/mask/mask.go @@ -7,7 +7,7 @@ import ( "slices" "strings" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) const maskSymbol = byte('*') diff --git a/internal/pkg/mask/mask_test.go b/internal/pkg/mask/mask_test.go index 612d237..152c4fe 100644 --- a/internal/pkg/mask/mask_test.go +++ b/internal/pkg/mask/mask_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestMaskCompile(t *testing.T) { diff --git a/internal/pkg/mask/masker.go b/internal/pkg/mask/masker.go index 7d75b26..4b1185f 100644 --- a/internal/pkg/mask/masker.go +++ b/internal/pkg/mask/masker.go @@ -3,7 +3,7 @@ package mask import ( "fmt" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) type Masker struct { diff --git a/internal/pkg/mask/masker_test.go b/internal/pkg/mask/masker_test.go index 3150786..c7b64f4 100644 --- a/internal/pkg/mask/masker_test.go +++ b/internal/pkg/mask/masker_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestMaskerMask(t *testing.T) { diff --git a/internal/pkg/redisclient/client.go b/internal/pkg/redisclient/client.go index 00f5d7a..bd4edb8 100644 --- a/internal/pkg/redisclient/client.go +++ b/internal/pkg/redisclient/client.go @@ -6,7 +6,7 @@ import ( "github.com/redis/go-redis/v9" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func New(ctx context.Context, cfg *config.Redis) (*redis.Client, error) { diff --git a/internal/pkg/service/admin/roles_test.go b/internal/pkg/service/admin/roles_test.go index 5063219..5982b43 100644 --- a/internal/pkg/service/admin/roles_test.go +++ b/internal/pkg/service/admin/roles_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock "github.com/ozontech/seq-ui/internal/pkg/repository/mock" diff --git a/internal/pkg/service/admin/service.go b/internal/pkg/service/admin/service.go index bc398cb..c4e51d4 100644 --- a/internal/pkg/service/admin/service.go +++ b/internal/pkg/service/admin/service.go @@ -3,7 +3,7 @@ package admin import ( "context" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/repository" diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index f7546c2..ea37a1e 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -11,7 +11,7 @@ import ( "go.uber.org/zap" "github.com/ozontech/seq-ui/internal/api/httputil" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/repository" diff --git a/internal/pkg/service/errorgroups/service.go b/internal/pkg/service/errorgroups/service.go index b537bde..2c9e6c0 100644 --- a/internal/pkg/service/errorgroups/service.go +++ b/internal/pkg/service/errorgroups/service.go @@ -8,7 +8,7 @@ import ( "golang.org/x/sync/errgroup" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" repositorych "github.com/ozontech/seq-ui/internal/pkg/repository_ch" ) diff --git a/internal/pkg/service/errorgroups/service_test.go b/internal/pkg/service/errorgroups/service_test.go index 49cc37d..df8e4f0 100644 --- a/internal/pkg/service/errorgroups/service_test.go +++ b/internal/pkg/service/errorgroups/service_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" mock "github.com/ozontech/seq-ui/internal/pkg/repository_ch/mock" ) diff --git a/internal/pkg/service/massexport/filestore/s3.go b/internal/pkg/service/massexport/filestore/s3.go index 2903e6e..d4f5048 100644 --- a/internal/pkg/service/massexport/filestore/s3.go +++ b/internal/pkg/service/massexport/filestore/s3.go @@ -11,7 +11,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) type s3FileStore struct { diff --git a/internal/pkg/service/massexport/seq_proxy_downloader.go b/internal/pkg/service/massexport/seq_proxy_downloader.go index 6bb6258..5823e14 100644 --- a/internal/pkg/service/massexport/seq_proxy_downloader.go +++ b/internal/pkg/service/massexport/seq_proxy_downloader.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/metric" diff --git a/internal/pkg/service/massexport/service.go b/internal/pkg/service/massexport/service.go index 03256c9..ee62510 100644 --- a/internal/pkg/service/massexport/service.go +++ b/internal/pkg/service/massexport/service.go @@ -7,7 +7,7 @@ import ( "slices" "time" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/service/massexport/filestore" diff --git a/internal/pkg/service/massexport/sessionstore/redis.go b/internal/pkg/service/massexport/sessionstore/redis.go index ebe503e..c4b245f 100644 --- a/internal/pkg/service/massexport/sessionstore/redis.go +++ b/internal/pkg/service/massexport/sessionstore/redis.go @@ -11,7 +11,7 @@ import ( "github.com/redis/go-redis/v9" "go.uber.org/zap" - config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/redisclient" "github.com/ozontech/seq-ui/logger" From e9f218501e5b2e0b00cf0ab335bc8c80fe138f3a Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 2 Sep 2026 15:51:57 +0300 Subject: [PATCH 13/29] create normalize file in package config, fix merge options, refactor config v2 scheme --- cmd/config-migrate/main.go | 3 +- cmd/seq-ui/main.go | 18 +- internal/api/seqapi/v1/test/data.go | 16 +- internal/app/config/loader/loader.go | 20 +- internal/app/config/migrate/v1_v2.go | 68 ++-- internal/app/config/v2/config.go | 588 ++++++++------------------- internal/app/config/v2/normalize.go | 343 ++++++++++++++++ 7 files changed, 572 insertions(+), 484 deletions(-) create mode 100644 internal/app/config/v2/normalize.go diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go index f8f99e5..6cc2abc 100644 --- a/cmd/config-migrate/main.go +++ b/cmd/config-migrate/main.go @@ -42,7 +42,8 @@ func run(source string) { if errors.Is(err, loader.ErrAlreadyLatestConfigVersion) { logger.Info("source config is already at the latest schema version, nothing to migrate", zap.String("source", source)) return - } else if err != nil { + } + if err != nil { logger.Fatal("migrate config", zap.String("source", source), zap.Error(err)) } diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index e1b24ad..08fafd2 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -150,10 +150,10 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac } var ( - adminV1 *admin_v1.Admin - asyncSearchesService asyncsearches.Service - userProfileV1 *userprofile_v1.UserProfile - dashboardsV1 *dashboards_v1.Dashboards + adminV1 *admin_v1.Admin + asyncSearchesSvc asyncsearches.Service + userProfileV1 *userprofile_v1.UserProfile + dashboardsV1 *dashboards_v1.Dashboards ) if db != nil { repo := repository.New(db, cfg.DB.RequestTimeout) @@ -164,7 +164,9 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac userProfileV1 = userprofile_v1.New(userProfilesSvc) dashboardsV1 = dashboards_v1.New(dashboardsSvc) - asyncSearchesService = asyncsearches.New(ctx, repo, seqDBClients[cfg.Handlers.AsyncSearch.SeqDBID], cfg.Handlers.AsyncSearch) + if cfg.Handlers.AsyncSearch != nil { + asyncSearchesSvc = asyncsearches.New(ctx, repo, seqDBClients[cfg.Handlers.AsyncSearch.SeqDBID], *cfg.Handlers.AsyncSearch) + } if cfg.Handlers.Admin != nil { logger.Info("initializing redis admin cache") @@ -181,7 +183,7 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac seqApiCache := caches[cfg.Handlers.SeqAPI.CacheID] seqApiRedisCache := caches[cfg.Handlers.SeqAPI.RedisID] - seqApiV1 := seqapi_v1.New(&cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesService) + seqApiV1 := seqapi_v1.New(&cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesSvc) var errorGroupsV1 *errorgroups_v1.ErrorGroups if cfg.Handlers.ErrorGroups != nil { @@ -209,12 +211,12 @@ func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb. } func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *config.SeqAPI) (seqdb.Client, error) { - maxSearchLimit := int(seqAPI.Options.MaxSearchLimit) + maxSearchLimit := int(seqAPI.Options.Limits.MaxSearchLimit) for _, env := range seqAPI.Envs { if env.SeqDBID != cfg.ID { continue } - if l := int(env.Options.MaxSearchLimit); l > maxSearchLimit { + if l := int(env.Options.Limits.MaxSearchLimit); l > maxSearchLimit { maxSearchLimit = l } } diff --git a/internal/api/seqapi/v1/test/data.go b/internal/api/seqapi/v1/test/data.go index 9af766c..6ca4084 100644 --- a/internal/api/seqapi/v1/test/data.go +++ b/internal/api/seqapi/v1/test/data.go @@ -111,17 +111,17 @@ func MakeAggregations(aggCount, bucketCount int, opts *MakeAggOpts) []*seqapi.Ag } func SetCfgDefaults(cfg config.SeqAPI) config.SeqAPI { - if cfg.Options.MaxAggregationsPerRequest <= 0 { - cfg.Options.MaxAggregationsPerRequest = 1 + if cfg.Options.Limits.MaxAggregationsPerRequest <= 0 { + cfg.Options.Limits.MaxAggregationsPerRequest = 1 } - if cfg.Options.MaxParallelExportRequests <= 0 { - cfg.Options.MaxParallelExportRequests = 1 + if cfg.Options.Limits.MaxParallelExportRequests <= 0 { + cfg.Options.Limits.MaxParallelExportRequests = 1 } - if cfg.Options.MaxSearchTotalLimit <= 0 { - cfg.Options.MaxSearchTotalLimit = 1000000 + if cfg.Options.Limits.MaxSearchTotal <= 0 { + cfg.Options.Limits.MaxSearchTotal = 1000000 } - if cfg.Options.MaxSearchOffsetLimit <= 0 { - cfg.Options.MaxSearchOffsetLimit = 1000000 + if cfg.Options.Limits.MaxSearchOffset <= 0 { + cfg.Options.Limits.MaxSearchOffset = 1000000 } for envName, envConfig := range cfg.Envs { diff --git a/internal/app/config/loader/loader.go b/internal/app/config/loader/loader.go index 8cf715d..29af649 100644 --- a/internal/app/config/loader/loader.go +++ b/internal/app/config/loader/loader.go @@ -128,14 +128,14 @@ func encode(cfg v2.Config) ([]byte, error) { } func mergeHandlersEnvOptions(cfgBytes []byte) ([]byte, error) { - var root map[any]any + var root map[string]any if err := yaml.Unmarshal(cfgBytes, &root); err != nil { return nil, fmt.Errorf("parse yaml for merge: %w", err) } handlers := getMap(root, "handlers") for _, h := range handlers { - handler, _ := h.(map[any]any) + handler, _ := h.(map[string]any) if handler == nil { continue } @@ -147,12 +147,12 @@ func mergeHandlersEnvOptions(cfgBytes []byte) ([]byte, error) { } for _, e := range envs { - env, _ := e.(map[any]any) + env, _ := e.(map[string]any) if env == nil { continue } - envOpts, _ := env["options"].(map[any]any) + envOpts, _ := env["options"].(map[string]any) env["options"] = mergeYAMLs(rootOpts, envOpts) } } @@ -164,10 +164,10 @@ func mergeHandlersEnvOptions(cfgBytes []byte) ([]byte, error) { return out, nil } -func getMap(root map[any]any, path ...string) map[any]any { +func getMap(root map[string]any, path ...string) map[string]any { cur := root for _, p := range path { - next, _ := cur[p].(map[any]any) + next, _ := cur[p].(map[string]any) if next == nil { return nil } @@ -177,14 +177,14 @@ func getMap(root map[any]any, path ...string) map[any]any { return cur } -func mergeYAMLs(a, b map[any]any) map[any]any { - merged := make(map[any]any) +func mergeYAMLs(a, b map[string]any) map[string]any { + merged := make(map[string]any) maps.Copy(merged, a) for k, v := range b { if existingValue, exists := merged[k]; exists { - if existingMap, ok := existingValue.(map[any]any); ok { - if newMap, ok := v.(map[any]any); ok { + if existingMap, ok := existingValue.(map[string]any); ok { + if newMap, ok := v.(map[string]any); ok { merged[k] = mergeYAMLs(existingMap, newMap) continue } diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index e177ffd..e2b281a 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -281,10 +281,14 @@ func migrateHandlers(src *v1.Handlers, cfg *v2.Config, v1Cache v1.Cache) *v2.Han func migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { cacheID, redisID := defaultCacheIDs(v1Cache) dst := v2.SeqAPI{ - CacheID: cacheID, - RedisID: redisID, - Envs: migrateSeqAPIEnvs(src.Envs), + Envs: migrateSeqAPIEnvs(src.Envs, cacheID, redisID), DefaultEnv: src.DefaultEnv, + Options: v2.SeqAPIOptions{ + Caches: v2.SeqAPICaches{ + CacheID: cacheID, + RedisID: redisID, + }, + }, } if len(src.Envs) == 0 { @@ -292,14 +296,13 @@ func migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { } if src.SeqAPIOptions != nil { - dst.GlobalOptions = migrateSeqAPIOptionsToGlobal(src.SeqAPIOptions) - dst.Options = migrateSeqAPIOptions(src.SeqAPIOptions) + dst.Options = migrateSeqAPIOptions(src.SeqAPIOptions, cacheID, redisID) } return dst } -func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv) map[string]v2.SeqAPIEnv { +func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv, cacheID, redisID string) map[string]v2.SeqAPIEnv { if len(envs) == 0 { return nil } @@ -310,7 +313,7 @@ func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv) map[string]v2.SeqAPIEnv { SeqDBID: cfg.SeqDB, } if cfg.Options != nil { - envOptions := migrateSeqAPIOptions(cfg.Options) + envOptions := migrateSeqAPIOptions(cfg.Options, cacheID, redisID) env.Options = &envOptions } @@ -320,27 +323,30 @@ func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv) map[string]v2.SeqAPIEnv { return dst } -func migrateSeqAPIOptionsToGlobal(options *v1.SeqAPIOptions) v2.SeqAPIGlobalOptions { - return v2.SeqAPIGlobalOptions{ - EventsCacheTTL: options.EventsCacheTTL, - PinnedFields: migrateFields(options.PinnedFields), - SystemFields: migrateFields(options.SystemFields), - LogsLifespanCacheTTL: options.LogsLifespanCacheTTL, - FieldsCacheTTL: options.FieldsCacheTTL, - Masking: migrateMasking(options.Masking), - } -} - -func migrateSeqAPIOptions(options *v1.SeqAPIOptions) v2.SeqAPIOptions { +func migrateSeqAPIOptions(options *v1.SeqAPIOptions, cacheID, redisID string) v2.SeqAPIOptions { return v2.SeqAPIOptions{ - MaxSearchLimit: options.MaxSearchLimit, - MaxSearchTotalLimit: options.MaxSearchTotalLimit, - MaxSearchOffsetLimit: options.MaxSearchOffsetLimit, - MaxExportLimit: options.MaxExportLimit, - SeqCLIMaxSearchLimit: options.SeqCLIMaxSearchLimit, - MaxParallelExportRequests: options.MaxParallelExportRequests, - MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, - MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, + Limits: v2.SeqAPILimits{ + MaxSearchLimit: options.MaxSearchLimit, + MaxSearchTotal: options.MaxSearchTotalLimit, + MaxSearchOffset: options.MaxSearchOffsetLimit, + MaxExportLimit: options.MaxExportLimit, + SeqCLIMaxSearchLimit: options.SeqCLIMaxSearchLimit, + MaxParallelExportRequests: options.MaxParallelExportRequests, + MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, + MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, + }, + Masking: migrateMasking(options.Masking), + PinnedFields: migrateFields(options.PinnedFields), + SystemFields: migrateFields(options.SystemFields), + Caches: v2.SeqAPICaches{ + CacheID: cacheID, + RedisID: redisID, + TTL: v2.SeqAPICachesTTL{ + Events: options.EventsCacheTTL, + LogsLifespan: options.LogsLifespanCacheTTL, + Fields: options.FieldsCacheTTL, + }, + }, } } @@ -433,8 +439,12 @@ func migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { } } -func migrateAsyncSearch(a v1.AsyncSearch) v2.AsyncSearch { - return v2.AsyncSearch{ +func migrateAsyncSearch(a v1.AsyncSearch) *v2.AsyncSearch { + if len(a.AdminUsers) == 0 && a.ListQueryLengthLimit == 0 { + return nil + } + + return &v2.AsyncSearch{ SeqDBID: v2.DefaultSeqDBClientID, AdminUsers: a.AdminUsers, ListQueryLengthLimit: a.ListQueryLengthLimit, diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index f02060f..d8f2ac5 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -27,32 +27,42 @@ import ( // debug: // addr: // auth: -// oidc: -// cache_id: -// cache_secret_key: -// skip_verify: -// auth_urls: -// tls: -// root_ca: -// ca_cert: -// private_key: -// insecure: -// allowed_clients: -// jwt: -// secret_key: +// options: +// oidc: +// cache_id: +// cache_secret_key: +// skip_verify: +// auth_urls: +// tls: +// root_ca: +// ca_cert: +// private_key: +// insecure: +// allowed_clients: +// jwt: +// secret_key: +// envs: +// : +// options: +// oidc: +// jwt: // rate_limiters: -// : -// default: -// rate_per_sec: -// max_burst: -// store_max_keys: -// per_handler: -// spec_users: -// : +// options: +// : +// default: // rate_per_sec: // max_burst: // store_max_keys: // per_handler: +// spec_users: +// : +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// envs: +// : +// options: // clients: // seq_db: // - id: @@ -83,18 +93,16 @@ import ( // handlers: // seq_api: // seq_db_id: -// cache_id: -// redis_id: -// global_options: -// events_cache_ttl: -// pinned_fields: -// - name: -// type: -// system_fields: -// - name: -// type: -// logs_lifespan_cache_ttl: -// fields_cache_ttl: +// options: +// limits: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: // masking: // masks: // - re: @@ -111,40 +119,38 @@ import ( // values: // process_fields: // ignore_fields: -// options: -// max_search_limit: -// max_search_total_limit: -// max_search_offset_limit: -// max_export_limit: -// seq_cli_max_search_limit: -// max_parallel_export_requests: -// max_aggregations_per_request: -// max_buckets_per_aggregation_ts: +// caches: +// cache_id: +// redis_id: +// ttl: +// events: +// logs_lifespan: +// fields: +// pinned_fields: +// - name: +// type: +// system_fields: +// - name: +// type: // envs: // : // seq_db_id: // options: -// max_search_limit: -// max_search_total_limit: -// max_search_offset_limit: -// max_export_limit: -// seq_cli_max_search_limit: -// max_parallel_export_requests: -// max_aggregations_per_request: -// max_buckets_per_aggregation_ts: // default_env: // error_groups: // ch_id: +// options: +// log_tags_mapping: +// env: +// service: +// release: +// query_filter: +// : // envs: // : // ch_id: +// options: // default_env: -// log_tags_mapping: -// env: -// service: -// release: -// query_filter: -// : // mass_export: // seq_db_id: // batch_size: @@ -169,17 +175,26 @@ import ( // max_retry_backoff: // async_search: // seq_db_id: +// options: +// admin_users: +// list_query_length_limit: // envs: // : // seq_db_id: -// list_query_length_limit: +// options: // default_env: -// admin_users: -// list_query_length_limit: // admin: -// redis_id: -// super_users: -// cache_ttl: +// options: +// redis_id: +// super_users: +// cache_ttl: +// envs: +// : +// options: +// redis_id: +// super_users: +// cache_ttl: +// default_env: // db: // name: // host: @@ -197,8 +212,8 @@ import ( // buffer_items: // redis: // - id: -// key_prefix: // with_inmem_id: +// key_prefix: // addr: // username: // password: @@ -207,55 +222,6 @@ import ( // min_retry_backoff: // max_retry_backoff: -const ( - DefaultSeqDBClientID = "default_seq_db" - DefaultCHClientID = "default_ch" - DefaultInmemCacheID = "default_inmem_cache" - DefaultRedisID = "default_redis" - DefaultRedis2ID = "default_redis_2" - DefaultMassExportRedisID = "mass_export" - - SeqDBClientModeGRPC = "grpc" - - MaskModeMask = "mask" - MaskModeReplace = "replace" - MaskModeCut = "cut" - - FieldFilterConditionAnd = "and" - FieldFilterConditionOr = "or" - FieldFilterConditionNot = "not" - - FieldFilterModeEqual = "equal" - FieldFilterModeContains = "contains" - FieldFilterModePrefix = "prefix" - FieldFilterModeSuffix = "suffix" - - minGRPCKeepaliveTime = 10 * time.Second - minGRPCKeepaliveTimeout = 1 * time.Second - - defaultAsyncSearchListQueryLengthLimit = 1000 - - defaultMaxSearchTotalLimit = 1000000 - defaultMaxSearchOffsetLimit = 1000000 - defaultMaxExportLimit = 100000 - defaultMaxAggregationsPerRequest = 1 - defaultMaxBucketsPerAggregationTs = 200 - defaultMaxParallelExportRequests = 1 - - defaultInmemCacheNumCounters = 10000000 - defaultInmemCacheMaxCost = 1000000 - defaultInmemCacheBufferItems = 64 - - defaultEventsCacheTTL = 24 * time.Hour - - defaultLogsLifespanCacheTTL = 10 * time.Minute - - defaultAdminCacheTTL = time.Hour - - defaultClickHouseDialTimeout = 5 * time.Second - defaultClickHouseReadTimeout = 30 * time.Second -) - type Config struct { Version int `yaml:"version"` Server *Server `yaml:"server"` @@ -333,8 +299,8 @@ type InmemoryCache struct { type Redis struct { ID string `yaml:"id"` - KeyPrefix string `yaml:"key_prefix"` WithInmemID string `yaml:"with_inmem_id"` + KeyPrefix string `yaml:"key_prefix"` Addr string `yaml:"addr"` Username string `yaml:"username"` Password string `yaml:"password"` @@ -435,16 +401,34 @@ type JWT struct { } type Auth struct { + Options AuthOptions `yaml:"options"` + Envs map[string]AuthEnv `yaml:"envs"` +} + +type AuthEnv struct { + Options *AuthOptions `yaml:"options"` +} + +type AuthOptions struct { OIDC *OIDC `yaml:"oidc"` JWT *JWT `yaml:"jwt"` } type Server struct { - HTTP HTTP `yaml:"http"` - GRPC GRPC `yaml:"grpc"` - Debug Debug `yaml:"debug"` - Auth *Auth `yaml:"auth"` - RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` + HTTP HTTP `yaml:"http"` + GRPC GRPC `yaml:"grpc"` + Debug Debug `yaml:"debug"` + Auth *Auth `yaml:"auth"` + RateLimiters RateLimiters `yaml:"rate_limiters"` +} + +type RateLimiters struct { + Options ApiRateLimiters `yaml:"options"` + Envs map[string]RateLimitersEnv `yaml:"envs"` +} + +type RateLimitersEnv struct { + Options ApiRateLimiters `yaml:"options"` } type GRPCKeepaliveParams struct { @@ -507,13 +491,23 @@ func (c *Clients) ClickHouseByID(id string) *CHClient { type Handlers struct { SeqAPI SeqAPI `yaml:"seq_api"` - ErrorGroups *ErrorGroups `yaml:"error_groups"` MassExport *MassExport `yaml:"mass_export"` + AsyncSearch *AsyncSearch `yaml:"async_search"` + ErrorGroups *ErrorGroups `yaml:"error_groups"` Admin *Admin `yaml:"admin"` - AsyncSearch AsyncSearch `yaml:"async_search"` } type Admin struct { + Options *AdminOptions `yaml:"options"` + Envs map[string]AdminEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type AdminEnv struct { + Options *AdminOptions `yaml:"options"` +} + +type AdminOptions struct { RedisID string `yaml:"redis_id"` SuperUsers []string `yaml:"super_users"` CacheTTL time.Duration `yaml:"cache_ttl"` @@ -525,22 +519,10 @@ type Field struct { } type SeqAPI struct { - SeqDBID string `yaml:"seq_db_id"` - CacheID string `yaml:"cache_id"` - RedisID string `yaml:"redis_id"` - GlobalOptions SeqAPIGlobalOptions `yaml:"global_options"` - Options SeqAPIOptions `yaml:"options"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` -} - -type SeqAPIGlobalOptions struct { - PinnedFields []Field `yaml:"pinned_fields"` - SystemFields []Field `yaml:"system_fields"` - Masking *Masking `yaml:"masking"` - EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` - LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` - FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` + SeqDBID string `yaml:"seq_db_id"` + Options SeqAPIOptions `yaml:"options"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` } type SeqAPIEnv struct { @@ -549,9 +531,17 @@ type SeqAPIEnv struct { } type SeqAPIOptions struct { + Limits SeqAPILimits `yaml:"limits"` + Caches SeqAPICaches `yaml:"caches"` + Masking *Masking `yaml:"masking"` + PinnedFields []Field `yaml:"pinned_fields"` + SystemFields []Field `yaml:"system_fields"` +} + +type SeqAPILimits struct { MaxSearchLimit int32 `yaml:"max_search_limit,omitempty"` - MaxSearchTotalLimit int64 `yaml:"max_search_total_limit,omitempty"` - MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit,omitempty"` + MaxSearchTotal int64 `yaml:"max_search_total,omitempty"` + MaxSearchOffset int32 `yaml:"max_search_offset,omitempty"` MaxExportLimit int32 `yaml:"max_export_limit,omitempty"` SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit,omitempty"` MaxParallelExportRequests int `yaml:"max_parallel_export_requests,omitempty"` @@ -559,6 +549,18 @@ type SeqAPIOptions struct { MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts,omitempty"` } +type SeqAPICaches struct { + CacheID string `yaml:"cache_id"` // redis + inmem + RedisID string `yaml:"redis_id"` // redis only + TTL SeqAPICachesTTL `yaml:"ttl"` +} + +type SeqAPICachesTTL struct { + Events time.Duration `yaml:"events"` + LogsLifespan time.Duration `yaml:"logs_lifespan"` + Fields time.Duration `yaml:"fields"` +} + type Masking struct { Masks []Mask `yaml:"masks"` ProcessFields []string `yaml:"process_fields"` @@ -595,305 +597,35 @@ type LogTagsMapping struct { } type ErrorGroups struct { - CHID string `yaml:"ch_id"` - Envs map[string]ErrorGroupsEnv `yaml:"envs"` - LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` - QueryFilter map[string]string `yaml:"query_filter"` - DefaultEnv string `yaml:"default_env"` + CHID string `yaml:"ch_id"` + Options *ErrorGroupsOptions `yaml:"options"` + Envs map[string]ErrorGroupsEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` } type ErrorGroupsEnv struct { - CHID string `yaml:"ch_id"` + CHID string `yaml:"ch_id"` + Options *ErrorGroupsOptions `yaml:"options"` } -type AsyncSearch struct { - SeqDBID string `yaml:"seq_db_id"` - Envs map[string]AsyncSearchEnv `yaml:"envs"` - AdminUsers []string `yaml:"admin_users"` - ListQueryLengthLimit int `yaml:"list_query_length_limit"` - DefaultEnv string `yaml:"default_env"` +type ErrorGroupsOptions struct { + LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` + QueryFilter map[string]string `yaml:"query_filter"` } -type AsyncSearchEnv struct { - SeqDBID string `yaml:"seq_db_id"` - ListQueryLengthLimit int `yaml:"list_query_length_limit"` -} - -func Normalize(cfg *Config) error { - if len(cfg.Clients.SeqDB) == 0 { - return fmt.Errorf("clients.seq_db must contain at least one client") - } - - seqDBIDs := make(map[string]struct{}, len(cfg.Clients.SeqDB)) - for i := range cfg.Clients.SeqDB { - c := &cfg.Clients.SeqDB[i] - if c.ID == "" { - return fmt.Errorf("clients.seq_db[%d].id cannot be empty", i) - } - if _, ok := seqDBIDs[c.ID]; ok { - return fmt.Errorf("duplicate clients.seq_db.id %q", c.ID) - } - - seqDBIDs[c.ID] = struct{}{} - - if c.ClientMode == "" { - c.ClientMode = SeqDBClientModeGRPC - } else if c.ClientMode != SeqDBClientModeGRPC { - return fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, SeqDBClientModeGRPC) - } - - if c.GRPCKeepaliveParams != nil { - c.GRPCKeepaliveParams.Time = max(c.GRPCKeepaliveParams.Time, minGRPCKeepaliveTime) - c.GRPCKeepaliveParams.Timeout = max(c.GRPCKeepaliveParams.Timeout, minGRPCKeepaliveTimeout) - } - } - - chIDs := make(map[string]struct{}, len(cfg.Clients.ClickHouse)) - for i := range cfg.Clients.ClickHouse { - ch := &cfg.Clients.ClickHouse[i] - if ch.ID == "" { - return fmt.Errorf("clients.clickhouse[%d].id cannot be empty", i) - } - if _, ok := chIDs[ch.ID]; ok { - return fmt.Errorf("duplicate clients.clickhouse.id %q", ch.ID) - } - - chIDs[ch.ID] = struct{}{} - - if ch.DialTimeout <= 0 { - ch.DialTimeout = defaultClickHouseDialTimeout - } - if ch.ReadTimeout <= 0 { - ch.ReadTimeout = defaultClickHouseReadTimeout - } - } - - inmemIDs := make(map[string]struct{}, len(cfg.Cache.Inmemory)) - for i := range cfg.Cache.Inmemory { - inm := &cfg.Cache.Inmemory[i] - if inm.ID == "" { - return fmt.Errorf("cache.inmemory[%d].id cannot be empty", i) - } - if _, ok := inmemIDs[inm.ID]; ok { - return fmt.Errorf("duplicate cache.inmemory.id %q", inm.ID) - } - - inmemIDs[inm.ID] = struct{}{} - - if inm.NumCounters <= 0 { - inm.NumCounters = defaultInmemCacheNumCounters - } - if inm.MaxCost <= 0 { - inm.MaxCost = defaultInmemCacheMaxCost - } - if inm.BufferItems <= 0 { - inm.BufferItems = defaultInmemCacheBufferItems - } - } - - redisIDs := make(map[string]struct{}, len(cfg.Cache.Redis)) - for i := range cfg.Cache.Redis { - r := &cfg.Cache.Redis[i] - if r.ID == "" { - return fmt.Errorf("cache.redis[%d].id cannot be empty", i) - } - if _, ok := redisIDs[r.ID]; ok { - return fmt.Errorf("duplicate cache.redis.id %q", r.ID) - } - - redisIDs[r.ID] = struct{}{} - - if r.WithInmemID != "" { - if _, ok := inmemIDs[r.WithInmemID]; !ok { - return fmt.Errorf("cache.redis[%q].with_inmem_id %q not found in cache.inmemory", r.ID, r.WithInmemID) - } - } - } - - if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { - cfg.DB.UsePreparedStatements = new(bool) - *cfg.DB.UsePreparedStatements = true - } - - if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { - cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit - } - - setSeqAPIGlobalOptionsDefaults(&cfg.Handlers.SeqAPI.GlobalOptions) - setSeqAPIOptionsDefaults(&cfg.Handlers.SeqAPI.Options) - - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - if cfg.Handlers.SeqAPI.SeqDBID != "" { - return fmt.Errorf("handlers.seq_api.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") - } - if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return fmt.Errorf("handlers.seq_api.default_env must be specified when envs is used") - } - if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return fmt.Errorf("handlers.seq_api.default_env %q not found in envs", cfg.Handlers.SeqAPI.DefaultEnv) - } - - for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if envConfig.SeqDBID == "" { - return fmt.Errorf("handlers.seq_api.envs[%q].seq_db_id cannot be empty", envName) - } - if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) - } - } - } else { - if cfg.Handlers.SeqAPI.SeqDBID == "" { - return fmt.Errorf("handlers.seq_api.seq_db_id cannot be empty when envs is not used") - } - if _, ok := seqDBIDs[cfg.Handlers.SeqAPI.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.seq_api.seq_db_id %q", cfg.Handlers.SeqAPI.SeqDBID) - } - } - - if cfg.Handlers.MassExport != nil { - if cfg.Handlers.MassExport.SeqDBID == "" { - return fmt.Errorf("handlers.mass_export.seq_db_id cannot be empty") - } - if _, ok := seqDBIDs[cfg.Handlers.MassExport.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.mass_export.seq_db_id %q", cfg.Handlers.MassExport.SeqDBID) - } - - if cfg.Handlers.MassExport.SessionStore != nil { - if cfg.Handlers.MassExport.SessionStore.RedisID == "" { - return fmt.Errorf("handlers.mass_export.session_store.redis_id cannot be empty") - } - if _, ok := redisIDs[cfg.Handlers.MassExport.SessionStore.RedisID]; !ok { - return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) - } - } - } - - if len(cfg.Handlers.AsyncSearch.Envs) > 0 { - if cfg.Handlers.AsyncSearch.SeqDBID != "" { - return fmt.Errorf("handlers.async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") - } - if cfg.Handlers.AsyncSearch.DefaultEnv == "" { - return fmt.Errorf("handlers.async_search.default_env must be specified when using envs") - } - if _, ok := cfg.Handlers.AsyncSearch.Envs[cfg.Handlers.AsyncSearch.DefaultEnv]; !ok { - return fmt.Errorf("handlers.async_search.default_env %q not found in envs", cfg.Handlers.AsyncSearch.DefaultEnv) - } - - for envName, envConfig := range cfg.Handlers.AsyncSearch.Envs { - if envConfig.SeqDBID == "" { - return fmt.Errorf("handlers.async_search.envs[%q].seq_db_id cannot be empty", envName) - } - if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) - } - if envConfig.ListQueryLengthLimit <= 0 { - envConfig.ListQueryLengthLimit = cfg.Handlers.AsyncSearch.ListQueryLengthLimit - } - - cfg.Handlers.AsyncSearch.Envs[envName] = envConfig - } - } else { - if cfg.Handlers.AsyncSearch.SeqDBID == "" { - return fmt.Errorf("handlers.async_search.seq_db_id cannot be empty when envs is not used") - } - if _, ok := seqDBIDs[cfg.Handlers.AsyncSearch.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) - } - } - - if cfg.Handlers.ErrorGroups != nil { - eg := cfg.Handlers.ErrorGroups - if len(eg.Envs) > 0 { - if eg.CHID != "" { - return fmt.Errorf("handlers.error_groups.ch_id must be empty when envs is used. Put ch_id inside each env") - } - if eg.DefaultEnv == "" { - return fmt.Errorf("handlers.error_groups.default_env must be specified when using envs") - } - if _, ok := eg.Envs[eg.DefaultEnv]; !ok { - return fmt.Errorf("handlers.error_groups.default_env %q not found in envs", eg.DefaultEnv) - } - - for envName, envConfig := range eg.Envs { - if envConfig.CHID == "" { - return fmt.Errorf("handlers.error_groups.envs[%q].ch_id cannot be empty", envName) - } - if _, ok := chIDs[envConfig.CHID]; !ok { - return fmt.Errorf("unknown handlers.error_groups.envs[%q].ch_id %q", envName, envConfig.CHID) - } - } - } else { - if cfg.Handlers.ErrorGroups.CHID == "" { - return fmt.Errorf("handlers.error_groups.ch_id cannot be empty when envs is not used") - } - if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { - return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) - } - } - } - - if cfg.Server.Auth != nil && cfg.Server.Auth.OIDC != nil { - oidc := cfg.Server.Auth.OIDC - hasCacheKey := oidc.CacheSecretKey != "" - hasCacheID := oidc.CacheID != "" - - switch { - case hasCacheKey && !hasCacheID: - return fmt.Errorf("auth.oidc.cache_secret_key is set but auth.oidc.cache_id is empty") - case !hasCacheKey && hasCacheID: - return fmt.Errorf("auth.oidc.cache_id is set but auth.oidc.cache_secret_key is empty") - case hasCacheID && hasCacheKey: - if cfg.Cache.InmemByID(oidc.CacheID) == nil && cfg.Cache.RedisByID(oidc.CacheID) == nil { - return fmt.Errorf("auth.oidc.cache_id %q not found", oidc.CacheID) - } - } - } - - if cfg.Handlers.Admin != nil { - if cfg.Handlers.Admin.CacheTTL <= 0 { - cfg.Handlers.Admin.CacheTTL = defaultAdminCacheTTL - } - - if cfg.Handlers.Admin.RedisID != "" { - redisCfg := cfg.Cache.RedisByID(cfg.Handlers.Admin.RedisID) - if redisCfg == nil { - return fmt.Errorf("unknown handlers.admin.redis_id %q", cfg.Handlers.Admin.RedisID) - } - if redisCfg.WithInmemID != "" { - return fmt.Errorf("handlers.admin.redis_id %q: with_inmem_id is not allowed", cfg.Handlers.Admin.RedisID) - } - } - } - - return nil +type AsyncSearch struct { + SeqDBID string `yaml:"seq_db_id"` + Options AsyncSearchOptions `yaml:"options"` + Envs map[string]AsyncSearchEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` } -func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { - if options.MaxAggregationsPerRequest <= 0 { - options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest - } - if options.MaxBucketsPerAggregationTs <= 0 { - options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs - } - if options.MaxParallelExportRequests <= 0 { - options.MaxParallelExportRequests = defaultMaxParallelExportRequests - } - if options.MaxSearchTotalLimit <= 0 { - options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit - } - if options.MaxSearchOffsetLimit <= 0 { - options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit - } - if options.MaxExportLimit <= 0 { - options.MaxExportLimit = defaultMaxExportLimit - } +type AsyncSearchEnv struct { + SeqDBID string `yaml:"seq_db_id"` + Options *AsyncSearchOptions `yaml:"options"` } -func setSeqAPIGlobalOptionsDefaults(options *SeqAPIGlobalOptions) { - if options.EventsCacheTTL <= 0 { - options.EventsCacheTTL = defaultEventsCacheTTL - } - if options.LogsLifespanCacheTTL <= 0 { - options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL - } +type AsyncSearchOptions struct { + AdminUsers []string `yaml:"admin_users"` + ListQueryLengthLimit int `yaml:"list_query_length_limit"` } diff --git a/internal/app/config/v2/normalize.go b/internal/app/config/v2/normalize.go new file mode 100644 index 0000000..af58186 --- /dev/null +++ b/internal/app/config/v2/normalize.go @@ -0,0 +1,343 @@ +package config + +import ( + "fmt" + "time" +) + +const ( + DefaultSeqDBClientID = "default_seq_db" + DefaultCHClientID = "default_ch" + DefaultInmemCacheID = "default_inmem_cache" + DefaultRedisID = "default_redis" + DefaultRedis2ID = "default_redis_2" + DefaultMassExportRedisID = "mass_export" + + SeqDBClientModeGRPC = "grpc" + + MaskModeMask = "mask" + MaskModeReplace = "replace" + MaskModeCut = "cut" + + FieldFilterConditionAnd = "and" + FieldFilterConditionOr = "or" + FieldFilterConditionNot = "not" + + FieldFilterModeEqual = "equal" + FieldFilterModeContains = "contains" + FieldFilterModePrefix = "prefix" + FieldFilterModeSuffix = "suffix" + + minGRPCKeepaliveTime = 10 * time.Second + minGRPCKeepaliveTimeout = 1 * time.Second + + defaultAsyncSearchListQueryLengthLimit = 1000 + + defaultMaxSearchTotal = 1000000 + defaultMaxSearchOffset = 1000000 + defaultMaxExportLimit = 100000 + defaultMaxAggregationsPerRequest = 1 + defaultMaxBucketsPerAggregationTs = 200 + defaultMaxParallelExportRequests = 1 + + defaultInmemCacheNumCounters = 10000000 + defaultInmemCacheMaxCost = 1000000 + defaultInmemCacheBufferItems = 64 + + defaultEventsCacheTTL = 24 * time.Hour + + defaultLogsLifespanCacheTTL = 10 * time.Minute + + defaultAdminCacheTTL = time.Hour + + defaultClickHouseDialTimeout = 5 * time.Second + defaultClickHouseReadTimeout = 30 * time.Second +) + +func Normalize(cfg *Config) error { + if len(cfg.Clients.SeqDB) == 0 { + return fmt.Errorf("clients.seq_db must contain at least one client") + } + + seqDBIDs := make(map[string]struct{}, len(cfg.Clients.SeqDB)) + for i := range cfg.Clients.SeqDB { + c := &cfg.Clients.SeqDB[i] + if c.ID == "" { + return fmt.Errorf("clients.seq_db[%d].id cannot be empty", i) + } + if _, ok := seqDBIDs[c.ID]; ok { + return fmt.Errorf("duplicate clients.seq_db.id %q", c.ID) + } + + seqDBIDs[c.ID] = struct{}{} + + if c.ClientMode == "" { + c.ClientMode = SeqDBClientModeGRPC + } else if c.ClientMode != SeqDBClientModeGRPC { + return fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, SeqDBClientModeGRPC) + } + + if c.GRPCKeepaliveParams != nil { + c.GRPCKeepaliveParams.Time = max(c.GRPCKeepaliveParams.Time, minGRPCKeepaliveTime) + c.GRPCKeepaliveParams.Timeout = max(c.GRPCKeepaliveParams.Timeout, minGRPCKeepaliveTimeout) + } + } + + chIDs := make(map[string]struct{}, len(cfg.Clients.ClickHouse)) + for i := range cfg.Clients.ClickHouse { + ch := &cfg.Clients.ClickHouse[i] + if ch.ID == "" { + return fmt.Errorf("clients.clickhouse[%d].id cannot be empty", i) + } + if _, ok := chIDs[ch.ID]; ok { + return fmt.Errorf("duplicate clients.clickhouse.id %q", ch.ID) + } + + chIDs[ch.ID] = struct{}{} + + if ch.DialTimeout <= 0 { + ch.DialTimeout = defaultClickHouseDialTimeout + } + if ch.ReadTimeout <= 0 { + ch.ReadTimeout = defaultClickHouseReadTimeout + } + } + + inmemIDs := make(map[string]struct{}, len(cfg.Cache.Inmemory)) + for i := range cfg.Cache.Inmemory { + inm := &cfg.Cache.Inmemory[i] + if inm.ID == "" { + return fmt.Errorf("cache.inmemory[%d].id cannot be empty", i) + } + if _, ok := inmemIDs[inm.ID]; ok { + return fmt.Errorf("duplicate cache.inmemory.id %q", inm.ID) + } + + inmemIDs[inm.ID] = struct{}{} + + if inm.NumCounters <= 0 { + inm.NumCounters = defaultInmemCacheNumCounters + } + if inm.MaxCost <= 0 { + inm.MaxCost = defaultInmemCacheMaxCost + } + if inm.BufferItems <= 0 { + inm.BufferItems = defaultInmemCacheBufferItems + } + } + + redisIDs := make(map[string]struct{}, len(cfg.Cache.Redis)) + for i := range cfg.Cache.Redis { + r := &cfg.Cache.Redis[i] + if r.ID == "" { + return fmt.Errorf("cache.redis[%d].id cannot be empty", i) + } + if _, ok := redisIDs[r.ID]; ok { + return fmt.Errorf("duplicate cache.redis.id %q", r.ID) + } + + redisIDs[r.ID] = struct{}{} + + if r.WithInmemID != "" { + if _, ok := inmemIDs[r.WithInmemID]; !ok { + return fmt.Errorf("cache.redis[%q].with_inmem_id %q not found in cache.inmemory", r.ID, r.WithInmemID) + } + } + } + + if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { + cfg.DB.UsePreparedStatements = new(bool) + *cfg.DB.UsePreparedStatements = true + } + + setSeqAPIOptionsDefaults(&cfg.Handlers.SeqAPI.Options) + + if len(cfg.Handlers.SeqAPI.Envs) > 0 { + if cfg.Handlers.SeqAPI.SeqDBID != "" { + return fmt.Errorf("handlers.seq_api.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + } + if cfg.Handlers.SeqAPI.DefaultEnv == "" { + return fmt.Errorf("handlers.seq_api.default_env must be specified when envs is used") + } + if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { + return fmt.Errorf("handlers.seq_api.default_env %q not found in envs", cfg.Handlers.SeqAPI.DefaultEnv) + } + + for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { + if envConfig.SeqDBID == "" { + return fmt.Errorf("handlers.seq_api.envs[%q].seq_db_id cannot be empty", envName) + } + if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + } + + setSeqAPIOptionsDefaults(envConfig.Options) + } + } else { + if cfg.Handlers.SeqAPI.SeqDBID == "" { + return fmt.Errorf("handlers.seq_api.seq_db_id cannot be empty when envs is not used") + } + if _, ok := seqDBIDs[cfg.Handlers.SeqAPI.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.seq_api.seq_db_id %q", cfg.Handlers.SeqAPI.SeqDBID) + } + } + + if cfg.Handlers.MassExport != nil { + if cfg.Handlers.MassExport.SeqDBID == "" { + return fmt.Errorf("handlers.mass_export.seq_db_id cannot be empty") + } + if _, ok := seqDBIDs[cfg.Handlers.MassExport.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.mass_export.seq_db_id %q", cfg.Handlers.MassExport.SeqDBID) + } + + if cfg.Handlers.MassExport.SessionStore != nil { + if cfg.Handlers.MassExport.SessionStore.RedisID == "" { + return fmt.Errorf("handlers.mass_export.session_store.redis_id cannot be empty") + } + if _, ok := redisIDs[cfg.Handlers.MassExport.SessionStore.RedisID]; !ok { + return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) + } + } + } + + if cfg.Handlers.AsyncSearch != nil { + as := cfg.Handlers.AsyncSearch + + if as.ListQueryLengthLimit <= 0 { + as.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + } + + if len(as.Envs) > 0 { + if as.SeqDBID != "" { + return fmt.Errorf("handlers.async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + } + if as.DefaultEnv == "" { + return fmt.Errorf("handlers.async_search.default_env must be specified when using envs") + } + if _, ok := as.Envs[as.DefaultEnv]; !ok { + return fmt.Errorf("handlers.async_search.default_env %q not found in envs", as.DefaultEnv) + } + + for envName, envConfig := range as.Envs { + if envConfig.SeqDBID == "" { + return fmt.Errorf("handlers.async_search.envs[%q].seq_db_id cannot be empty", envName) + } + if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + } + if envConfig.ListQueryLengthLimit <= 0 { + envConfig.ListQueryLengthLimit = as.ListQueryLengthLimit + } + + as.Envs[envName] = envConfig + } + } else { + if as.SeqDBID == "" { + return fmt.Errorf("handlers.async_search.seq_db_id cannot be empty when envs is not used") + } + if _, ok := seqDBIDs[as.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", as.SeqDBID) + } + } + } + + if cfg.Handlers.ErrorGroups != nil { + eg := cfg.Handlers.ErrorGroups + if len(eg.Envs) > 0 { + if eg.CHID != "" { + return fmt.Errorf("handlers.error_groups.ch_id must be empty when envs is used. Put ch_id inside each env") + } + if eg.DefaultEnv == "" { + return fmt.Errorf("handlers.error_groups.default_env must be specified when using envs") + } + if _, ok := eg.Envs[eg.DefaultEnv]; !ok { + return fmt.Errorf("handlers.error_groups.default_env %q not found in envs", eg.DefaultEnv) + } + + for envName, envConfig := range eg.Envs { + if envConfig.CHID == "" { + return fmt.Errorf("handlers.error_groups.envs[%q].ch_id cannot be empty", envName) + } + if _, ok := chIDs[envConfig.CHID]; !ok { + return fmt.Errorf("unknown handlers.error_groups.envs[%q].ch_id %q", envName, envConfig.CHID) + } + } + } else { + if cfg.Handlers.ErrorGroups.CHID == "" { + return fmt.Errorf("handlers.error_groups.ch_id cannot be empty when envs is not used") + } + if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { + return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) + } + } + } + + if cfg.Server.Auth != nil && cfg.Server.Auth.OIDC != nil { + oidc := cfg.Server.Auth.OIDC + hasCacheKey := oidc.CacheSecretKey != "" + hasCacheID := oidc.CacheID != "" + + switch { + case hasCacheKey && !hasCacheID: + return fmt.Errorf("auth.oidc.cache_secret_key is set but auth.oidc.cache_id is empty") + case !hasCacheKey && hasCacheID: + return fmt.Errorf("auth.oidc.cache_id is set but auth.oidc.cache_secret_key is empty") + case hasCacheID && hasCacheKey: + if cfg.Cache.InmemByID(oidc.CacheID) == nil && cfg.Cache.RedisByID(oidc.CacheID) == nil { + return fmt.Errorf("auth.oidc.cache_id %q not found", oidc.CacheID) + } + } + } + + if cfg.Handlers.Admin != nil { + if cfg.Handlers.Admin.CacheTTL <= 0 { + cfg.Handlers.Admin.CacheTTL = defaultAdminCacheTTL + } + + if cfg.Handlers.Admin.RedisID != "" { + redisCfg := cfg.Cache.RedisByID(cfg.Handlers.Admin.RedisID) + if redisCfg == nil { + return fmt.Errorf("unknown handlers.admin.redis_id %q", cfg.Handlers.Admin.RedisID) + } + if redisCfg.WithInmemID != "" { + return fmt.Errorf("handlers.admin.redis_id %q: with_inmem_id is not allowed", cfg.Handlers.Admin.RedisID) + } + } + } + + return nil +} + +func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { + if options.Limits.MaxAggregationsPerRequest <= 0 { + options.Limits.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest + } + if options.Limits.MaxBucketsPerAggregationTs <= 0 { + options.Limits.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs + } + if options.Limits.MaxParallelExportRequests <= 0 { + options.Limits.MaxParallelExportRequests = defaultMaxParallelExportRequests + } + if options.Limits.MaxSearchTotal <= 0 { + options.Limits.MaxSearchTotal = defaultMaxSearchTotal + } + if options.Limits.MaxSearchOffset <= 0 { + options.Limits.MaxSearchOffset = defaultMaxSearchOffset + } + if options.Limits.MaxExportLimit <= 0 { + options.Limits.MaxExportLimit = defaultMaxExportLimit + } + if options.Limits.MaxSearchLimit <= 0 { + options.Limits.MaxSearchLimit = 1000 // Надо дефолт задать нам, до этого его не было + } + if options.Limits.SeqCLIMaxSearchLimit <= 0 { + options.Limits.SeqCLIMaxSearchLimit = 1000 // Сюда тоже или в них не было небходимости вообще, чтобы они были по нулям + } + + if options.Caches.TTL.Events <= 0 { + options.Caches.TTL.Events = defaultEventsCacheTTL + } + if options.Caches.TTL.LogsLifespan <= 0 { + options.Caches.TTL.LogsLifespan = defaultLogsLifespanCacheTTL + } +} From b558fe5488fd5240a1a63c4c1355f4d101b079c9 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Thu, 3 Sep 2026 23:23:32 +0300 Subject: [PATCH 14/29] refactor schema config --- internal/app/config/loader/loader.go | 4 +- internal/app/config/v1/config.go | 4 ++ internal/app/config/v2/config.go | 37 +++++++--------- internal/app/config/v2/normalize.go | 63 +++++++++++++--------------- 4 files changed, 52 insertions(+), 56 deletions(-) diff --git a/internal/app/config/loader/loader.go b/internal/app/config/loader/loader.go index 29af649..6099130 100644 --- a/internal/app/config/loader/loader.go +++ b/internal/app/config/loader/loader.go @@ -45,7 +45,7 @@ func FromFile(cfgPath string) (v2.Config, error) { } cfgBytes, err = encode(migrate.V1ToV2(cfgV1)) if err != nil { - return v2.Config{}, nil + return v2.Config{}, err } case V2: default: @@ -59,7 +59,7 @@ func FromFile(cfgPath string) (v2.Config, error) { cfg, err := parse[v2.Config](cfgBytes, true) if err != nil { - return v2.Config{}, fmt.Errorf("error parsing config v2: %w", err) + return v2.Config{}, fmt.Errorf("parse config v2: %w", err) } if err := v2.Normalize(&cfg); err != nil { diff --git a/internal/app/config/v1/config.go b/internal/app/config/v1/config.go index b1b34c8..ac75aa2 100644 --- a/internal/app/config/v1/config.go +++ b/internal/app/config/v1/config.go @@ -198,6 +198,7 @@ import ( // enable_ssl: // session_store: // redis: +// key_prefix // addr: // username: // password: @@ -213,6 +214,9 @@ import ( // async_search: // admin_users: // list_query_length_limit: +// admin: +// admin_users: +// cache_ttl: type Config struct { Version *int `yaml:"version"` diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index d8f2ac5..1680952 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -27,8 +27,8 @@ import ( // debug: // addr: // auth: -// options: -// oidc: +// options: +// oidc: // cache_id: // cache_secret_key: // skip_verify: @@ -43,9 +43,7 @@ import ( // secret_key: // envs: // : -// options: -// oidc: -// jwt: +// options: # same as root options // rate_limiters: // options: // : @@ -62,7 +60,7 @@ import ( // per_handler: // envs: // : -// options: +// options: # same as root options // clients: // seq_db: // - id: @@ -96,8 +94,8 @@ import ( // options: // limits: // max_search_limit: -// max_search_total_limit: -// max_search_offset_limit: +// max_search_total: +// max_search_offset: // max_export_limit: // seq_cli_max_search_limit: // max_parallel_export_requests: @@ -135,7 +133,7 @@ import ( // envs: // : // seq_db_id: -// options: +// options: # same as root options // default_env: // error_groups: // ch_id: @@ -149,7 +147,7 @@ import ( // envs: // : // ch_id: -// options: +// options: # same as root options // default_env: // mass_export: // seq_db_id: @@ -177,23 +175,20 @@ import ( // seq_db_id: // options: // admin_users: -// list_query_length_limit: +// list_query_length_limit: // envs: // : // seq_db_id: -// options: +// options: # same as root options // default_env: // admin: -// options: +// options: // redis_id: // super_users: // cache_ttl: // envs: // : -// options: -// redis_id: -// super_users: -// cache_ttl: +// options: # same as root options // default_env: // db: // name: @@ -423,12 +418,12 @@ type Server struct { } type RateLimiters struct { - Options ApiRateLimiters `yaml:"options"` + Options ApiToRateLimiters `yaml:"options"` Envs map[string]RateLimitersEnv `yaml:"envs"` } type RateLimitersEnv struct { - Options ApiRateLimiters `yaml:"options"` + Options ApiToRateLimiters `yaml:"options"` } type GRPCKeepaliveParams struct { @@ -498,7 +493,7 @@ type Handlers struct { } type Admin struct { - Options *AdminOptions `yaml:"options"` + Options AdminOptions `yaml:"options"` Envs map[string]AdminEnv `yaml:"envs"` DefaultEnv string `yaml:"default_env"` } @@ -598,7 +593,7 @@ type LogTagsMapping struct { type ErrorGroups struct { CHID string `yaml:"ch_id"` - Options *ErrorGroupsOptions `yaml:"options"` + Options ErrorGroupsOptions `yaml:"options"` Envs map[string]ErrorGroupsEnv `yaml:"envs"` DefaultEnv string `yaml:"default_env"` } diff --git a/internal/app/config/v2/normalize.go b/internal/app/config/v2/normalize.go index af58186..8cd58fb 100644 --- a/internal/app/config/v2/normalize.go +++ b/internal/app/config/v2/normalize.go @@ -150,6 +150,23 @@ func Normalize(cfg *Config) error { *cfg.DB.UsePreparedStatements = true } + if cfg.Server.Auth != nil && cfg.Server.Auth.Options.OIDC != nil { + oidc := cfg.Server.Auth.Options.OIDC + hasCacheKey := oidc.CacheSecretKey != "" + hasCacheID := oidc.CacheID != "" + + switch { + case hasCacheKey && !hasCacheID: + return fmt.Errorf("auth.oidc.cache_secret_key is set but auth.oidc.cache_id is empty") + case !hasCacheKey && hasCacheID: + return fmt.Errorf("auth.oidc.cache_id is set but auth.oidc.cache_secret_key is empty") + case hasCacheID && hasCacheKey: + if cfg.Cache.InmemByID(oidc.CacheID) == nil && cfg.Cache.RedisByID(oidc.CacheID) == nil { + return fmt.Errorf("auth.oidc.cache_id %q not found", oidc.CacheID) + } + } + } + setSeqAPIOptionsDefaults(&cfg.Handlers.SeqAPI.Options) if len(cfg.Handlers.SeqAPI.Envs) > 0 { @@ -170,7 +187,6 @@ func Normalize(cfg *Config) error { if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) } - setSeqAPIOptionsDefaults(envConfig.Options) } } else { @@ -203,8 +219,8 @@ func Normalize(cfg *Config) error { if cfg.Handlers.AsyncSearch != nil { as := cfg.Handlers.AsyncSearch - if as.ListQueryLengthLimit <= 0 { - as.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + if as.Options.ListQueryLengthLimit <= 0 { + as.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit } if len(as.Envs) > 0 { @@ -212,7 +228,7 @@ func Normalize(cfg *Config) error { return fmt.Errorf("handlers.async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") } if as.DefaultEnv == "" { - return fmt.Errorf("handlers.async_search.default_env must be specified when using envs") + return fmt.Errorf("handlers.async_search.default_env must be specified when envs is used") } if _, ok := as.Envs[as.DefaultEnv]; !ok { return fmt.Errorf("handlers.async_search.default_env %q not found in envs", as.DefaultEnv) @@ -225,11 +241,9 @@ func Normalize(cfg *Config) error { if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) } - if envConfig.ListQueryLengthLimit <= 0 { - envConfig.ListQueryLengthLimit = as.ListQueryLengthLimit + if envConfig.Options.ListQueryLengthLimit <= 0 { + envConfig.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit } - - as.Envs[envName] = envConfig } } else { if as.SeqDBID == "" { @@ -248,7 +262,7 @@ func Normalize(cfg *Config) error { return fmt.Errorf("handlers.error_groups.ch_id must be empty when envs is used. Put ch_id inside each env") } if eg.DefaultEnv == "" { - return fmt.Errorf("handlers.error_groups.default_env must be specified when using envs") + return fmt.Errorf("handlers.error_groups.default_env must be specified when envs is used") } if _, ok := eg.Envs[eg.DefaultEnv]; !ok { return fmt.Errorf("handlers.error_groups.default_env %q not found in envs", eg.DefaultEnv) @@ -272,35 +286,19 @@ func Normalize(cfg *Config) error { } } - if cfg.Server.Auth != nil && cfg.Server.Auth.OIDC != nil { - oidc := cfg.Server.Auth.OIDC - hasCacheKey := oidc.CacheSecretKey != "" - hasCacheID := oidc.CacheID != "" - - switch { - case hasCacheKey && !hasCacheID: - return fmt.Errorf("auth.oidc.cache_secret_key is set but auth.oidc.cache_id is empty") - case !hasCacheKey && hasCacheID: - return fmt.Errorf("auth.oidc.cache_id is set but auth.oidc.cache_secret_key is empty") - case hasCacheID && hasCacheKey: - if cfg.Cache.InmemByID(oidc.CacheID) == nil && cfg.Cache.RedisByID(oidc.CacheID) == nil { - return fmt.Errorf("auth.oidc.cache_id %q not found", oidc.CacheID) - } - } - } - if cfg.Handlers.Admin != nil { - if cfg.Handlers.Admin.CacheTTL <= 0 { - cfg.Handlers.Admin.CacheTTL = defaultAdminCacheTTL + admin := cfg.Handlers.Admin + if admin.Options.CacheTTL <= 0 { + admin.Options.CacheTTL = defaultAdminCacheTTL } - if cfg.Handlers.Admin.RedisID != "" { - redisCfg := cfg.Cache.RedisByID(cfg.Handlers.Admin.RedisID) + if admin.Options.RedisID != "" { + redisCfg := cfg.Cache.RedisByID(admin.Options.RedisID) if redisCfg == nil { - return fmt.Errorf("unknown handlers.admin.redis_id %q", cfg.Handlers.Admin.RedisID) + return fmt.Errorf("unknown handlers.admin.redis_id %q", admin.Options.RedisID) } if redisCfg.WithInmemID != "" { - return fmt.Errorf("handlers.admin.redis_id %q: with_inmem_id is not allowed", cfg.Handlers.Admin.RedisID) + return fmt.Errorf("handlers.admin.redis_id %q: with_inmem_id is not allowed", admin.Options.RedisID) } } } @@ -333,7 +331,6 @@ func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { if options.Limits.SeqCLIMaxSearchLimit <= 0 { options.Limits.SeqCLIMaxSearchLimit = 1000 // Сюда тоже или в них не было небходимости вообще, чтобы они были по нулям } - if options.Caches.TTL.Events <= 0 { options.Caches.TTL.Events = defaultEventsCacheTTL } From cf16c5cda5e2c8f5737043c399ef34ecdf142f57 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 16 Sep 2026 14:48:02 +0300 Subject: [PATCH 15/29] update config scheme --- internal/app/config/loader/loader.go | 37 ++--- internal/app/config/migrate/v1_v2.go | 198 +++++++++++++++------------ internal/app/config/v2/config.go | 101 +++++++------- internal/app/config/v2/normalize.go | 46 ++++--- 4 files changed, 205 insertions(+), 177 deletions(-) diff --git a/internal/app/config/loader/loader.go b/internal/app/config/loader/loader.go index 6099130..af41a6c 100644 --- a/internal/app/config/loader/loader.go +++ b/internal/app/config/loader/loader.go @@ -32,14 +32,14 @@ func FromFile(cfgPath string) (v2.Config, error) { return v2.Config{}, fmt.Errorf("read file: %w", err) } - version, err := ReadVersion(cfgBytes) + version, err := readVersion(cfgBytes) if err != nil { return v2.Config{}, fmt.Errorf("read version: %w", err) } switch version { case V1: - cfgV1, err := parse[v1.Config](cfgBytes, true) + cfgV1, err := decode[v1.Config](cfgBytes, true) if err != nil { return v2.Config{}, fmt.Errorf("parse config v1: %w", err) } @@ -57,12 +57,12 @@ func FromFile(cfgPath string) (v2.Config, error) { return v2.Config{}, fmt.Errorf("merge env options: %w", err) } - cfg, err := parse[v2.Config](cfgBytes, true) + cfg, err := decode[v2.Config](cfgBytes, true) if err != nil { return v2.Config{}, fmt.Errorf("parse config v2: %w", err) } - if err := v2.Normalize(&cfg); err != nil { + if err := cfg.Validate(); err != nil { return v2.Config{}, fmt.Errorf("normalize config: %w", err) } @@ -70,14 +70,14 @@ func FromFile(cfgPath string) (v2.Config, error) { } func ToLatestVersion(cfgBytes []byte) ([]byte, error) { - version, err := ReadVersion(cfgBytes) + version, err := readVersion(cfgBytes) if err != nil { return nil, fmt.Errorf("read version: %w", err) } switch version { case V1: - cfgV1, err := parse[v1.Config](cfgBytes, true) + cfgV1, err := decode[v1.Config](cfgBytes, true) if err != nil { return nil, fmt.Errorf("parse config v1: %w", err) } @@ -89,8 +89,8 @@ func ToLatestVersion(cfgBytes []byte) ([]byte, error) { } } -func ReadVersion(cfgBytes []byte) (int, error) { - meta, err := parse[configMeta](cfgBytes, false) +func readVersion(cfgBytes []byte) (int, error) { + meta, err := decode[configMeta](cfgBytes, false) if err != nil { return 0, err } @@ -102,7 +102,7 @@ func ReadVersion(cfgBytes []byte) (int, error) { return *meta.Version, nil } -func parse[T any](cfg []byte, strict bool) (T, error) { +func decode[T any](cfg []byte, strict bool) (T, error) { var result T decoder := yaml.NewDecoder(bytes.NewReader(cfg)) @@ -133,15 +133,15 @@ func mergeHandlersEnvOptions(cfgBytes []byte) ([]byte, error) { return nil, fmt.Errorf("parse yaml for merge: %w", err) } - handlers := getMap(root, "handlers") + handlers, _ := root["handlers"].(map[string]any) for _, h := range handlers { handler, _ := h.(map[string]any) if handler == nil { continue } - rootOpts := getMap(handler, "options") - envs := getMap(handler, "envs") + rootOpts := handler["options"].(map[string]any) + envs := handlers["envs"].(map[string]any) if rootOpts == nil || len(envs) == 0 { continue } @@ -164,19 +164,6 @@ func mergeHandlersEnvOptions(cfgBytes []byte) ([]byte, error) { return out, nil } -func getMap(root map[string]any, path ...string) map[string]any { - cur := root - for _, p := range path { - next, _ := cur[p].(map[string]any) - if next == nil { - return nil - } - cur = next - } - - return cur -} - func mergeYAMLs(a, b map[string]any) map[string]any { merged := make(map[string]any) maps.Copy(merged, a) diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index e2b281a..c76ab92 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -5,26 +5,35 @@ import ( v2 "github.com/ozontech/seq-ui/internal/app/config/v2" ) +type v1tov2 struct { + src v1.Config +} + func V1ToV2(src v1.Config) v2.Config { + m := v1tov2{src: src} + return m.migrate() +} + +func (m *v1tov2) migrate() v2.Config { dst := v2.Config{Version: 2} - dst.Server = migrateServer(src.Server) - dst.Cache = migrateCache(src.Server) - dst.Clients = migrateClients(src) - dst.DB = migrateDB(src.Server) - dst.Handlers = migrateHandlers(src.Handlers, &dst, src.Server.Cache) + dst.Server = m.migrateServer(m.src.Server) + dst.Cache = m.migrateCache(m.src.Server) + dst.Clients = m.migrateClients(m.src) + dst.DB = m.migrateDB(m.src.Server) + dst.Handlers = m.migrateHandlers(m.src.Handlers, &dst, m.src.Server.Cache) return dst } -func migrateServer(src *v1.Server) *v2.Server { +func (m *v1tov2) migrateServer(src *v1.Server) *v2.Server { dst := &v2.Server{ HTTP: v2.HTTP{ Addr: src.HTTPAddr, ReadTimeout: src.HTTPReadTimeout, ReadHeaderTimeout: src.HTTPReadHeaderTimeout, WriteTimeout: src.HTTPWriteTimeout, - CORS: migrateCORS(src.CORS), + CORS: m.migrateCORS(src.CORS), }, GRPC: v2.GRPC{ Addr: src.GRPCAddr, @@ -33,19 +42,19 @@ func migrateServer(src *v1.Server) *v2.Server { Debug: v2.Debug{ Addr: src.DebugAddr, }, - RateLimiters: migrateApiRateLimiters(src.RateLimiters), + RateLimiters: m.migrateApiRateLimiters(src.RateLimiters), } if src.OIDC != nil { oidc := &v2.OIDC{ SkipVerify: src.OIDC.SkipVerify, AuthURLs: src.OIDC.AuthURLs, - TLS: migrateTLS(src.OIDC), + TLS: m.migrateTLS(src.OIDC), AllowedClients: src.OIDC.AllowedClients, } if src.OIDC.CacheSecretKey != "" { - cacheID, _ := defaultCacheIDs(src.Cache) + cacheID, _ := m.defaultCacheIDs(src.Cache) oidc.CacheID = cacheID } @@ -62,21 +71,21 @@ func migrateServer(src *v1.Server) *v2.Server { return dst } -func migrateTLS(src *v1.OIDC) *v2.TLS { +func (m *v1tov2) migrateTLS(src *v1.OIDC) *v2.TLS { if src.RootCA == "" && src.CACert == "" && src.PrivateKey == "" && !src.SSLSkipVerify { return nil } return &v2.TLS{ - RootCA: src.RootCA, - CACert: src.CACert, - PrivateKey: src.PrivateKey, + CACert: src.RootCA, + ClientCert: src.CACert, + ClientKey: src.PrivateKey, Insecure: src.SSLSkipVerify, } } -func migrateCORS(src *v1.CORS) *v2.CORS { +func (m *v1tov2) migrateCORS(src *v1.CORS) *v2.CORS { if src == nil { return nil } @@ -92,7 +101,7 @@ func migrateCORS(src *v1.CORS) *v2.CORS { } } -func migrateApiRateLimiters(src v1.ApiToRateLimiters) v2.ApiToRateLimiters { +func (m *v1tov2) migrateApiRateLimiters(src v1.ApiToRateLimiters) v2.ApiToRateLimiters { if src == nil { return nil } @@ -100,28 +109,28 @@ func migrateApiRateLimiters(src v1.ApiToRateLimiters) v2.ApiToRateLimiters { dst := make(v2.ApiToRateLimiters, len(src)) for api, rl := range src { dst[api] = v2.ApiRateLimiters{ - Default: migrateRateLimiter(rl.Default), - SpecialUsers: migrateUserToRateLimiter(rl.SpecialUsers), + Default: m.migrateRateLimiter(rl.Default), + SpecialUsers: m.migrateUserToRateLimiter(rl.SpecialUsers), } } return dst } -func migrateUserToRateLimiter(src v1.UserToRateLimiter) v2.UserToRateLimiter { +func (m *v1tov2) migrateUserToRateLimiter(src v1.UserToRateLimiter) v2.UserToRateLimiter { if src == nil { return nil } dst := make(v2.UserToRateLimiter, len(src)) for k, rl := range src { - dst[k] = migrateRateLimiter(rl) + dst[k] = m.migrateRateLimiter(rl) } return dst } -func migrateRateLimiter(src v1.RateLimiter) v2.RateLimiter { +func (m *v1tov2) migrateRateLimiter(src v1.RateLimiter) v2.RateLimiter { return v2.RateLimiter{ RatePerSec: src.RatePerSec, MaxBurst: src.MaxBurst, @@ -130,26 +139,28 @@ func migrateRateLimiter(src v1.RateLimiter) v2.RateLimiter { } } -func migrateGRPCKeepaliveParams(src *v1.GRPCKeepaliveParams) *v2.GRPCKeepaliveParams { +func (m *v1tov2) migrateGRPCParams(src *v1.GRPCKeepaliveParams) *v2.GRPCParams { if src == nil { return nil } - return &v2.GRPCKeepaliveParams{ - Time: src.Time, - Timeout: src.Timeout, - PermitWithoutStream: src.PermitWithoutStream, + return &v2.GRPCParams{ + Keepalive: &v2.KeepaliveParams{ + Time: src.Time, + Timeout: src.Timeout, + PermitWithoutStream: src.PermitWithoutStream, + }, } } -func migrateClients(src v1.Config) *v2.Clients { +func (m *v1tov2) migrateClients(src v1.Config) *v2.Clients { dst := &v2.Clients{} if src.Clients != nil { if len(src.Clients.SeqDB) > 0 { dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) for i := range src.Clients.SeqDB { - dst.SeqDB = append(dst.SeqDB, migrateSeqDBClient(&src.Clients.SeqDB[i])) + dst.SeqDB = append(dst.SeqDB, m.migrateSeqDBClient(&src.Clients.SeqDB[i])) } } else { dst.SeqDB = []v2.SeqDBClient{{ @@ -161,7 +172,7 @@ func migrateClients(src v1.Config) *v2.Clients { InitialRetryBackoff: src.Clients.InitialRetryBackoff, MaxRetryBackoff: src.Clients.MaxRetryBackoff, ClientMode: src.Clients.ProxyClientMode, - GRPCKeepaliveParams: migrateGRPCKeepaliveParams(src.Clients.GRPCKeepaliveParams), + GRPCParams: m.migrateGRPCParams(src.Clients.GRPCKeepaliveParams), }} } } @@ -183,7 +194,7 @@ func migrateClients(src v1.Config) *v2.Clients { return dst } -func migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { +func (m *v1tov2) migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { return v2.SeqDBClient{ ID: src.ID, Timeout: src.Timeout, @@ -193,11 +204,11 @@ func migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { InitialRetryBackoff: src.InitialRetryBackoff, MaxRetryBackoff: src.MaxRetryBackoff, ClientMode: src.ClientMode, - GRPCKeepaliveParams: migrateGRPCKeepaliveParams(src.GRPCKeepaliveParams), + GRPCParams: m.migrateGRPCParams(src.GRPCKeepaliveParams), } } -func migrateDB(src *v1.Server) *v2.DB { +func (m *v1tov2) migrateDB(src *v1.Server) *v2.DB { if src == nil || src.DB == nil { return nil } @@ -214,14 +225,14 @@ func migrateDB(src *v1.Server) *v2.DB { } } -func migrateCache(src *v1.Server) *v2.Cache { +func (m *v1tov2) migrateCache(src *v1.Server) *v2.Cache { if src == nil { return &v2.Cache{} } cache := &v2.Cache{} - if !isInmemEmpty(src.Cache.Inmemory) { + if !m.isInmemEmpty(src.Cache.Inmemory) { cache.Inmemory = append(cache.Inmemory, v2.InmemoryCache{ ID: v2.DefaultInmemCacheID, NumCounters: src.Cache.Inmemory.NumCounters, @@ -233,18 +244,18 @@ func migrateCache(src *v1.Server) *v2.Cache { if src.Cache.Redis != nil { if len(cache.Inmemory) > 0 { cache.Redis = append(cache.Redis, - migrateRedis(src.Cache.Redis, v2.DefaultRedisID, v2.DefaultInmemCacheID), - migrateRedis(src.Cache.Redis, v2.DefaultRedis2ID, ""), + m.migrateRedis(src.Cache.Redis, v2.DefaultRedisID, v2.DefaultInmemCacheID), + m.migrateRedis(src.Cache.Redis, v2.DefaultRedis2ID, ""), ) } else { - cache.Redis = append(cache.Redis, migrateRedis(src.Cache.Redis, v2.DefaultRedisID, "")) + cache.Redis = append(cache.Redis, m.migrateRedis(src.Cache.Redis, v2.DefaultRedisID, "")) } } return cache } -func migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { +func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { return v2.Redis{ ID: id, WithInmemID: withInmemID, @@ -259,34 +270,44 @@ func migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { } } -func migrateHandlers(src *v1.Handlers, cfg *v2.Config, v1Cache v1.Cache) *v2.Handlers { +func (m *v1tov2) migrateHandlers(src *v1.Handlers, cfg *v2.Config, v1Cache v1.Cache) *v2.Handlers { if src == nil { return &v2.Handlers{} } dst := &v2.Handlers{ - SeqAPI: migrateSeqAPI(src.SeqAPI, v1Cache), - ErrorGroups: migrateErrorGroups(src.ErrorGroups), - AsyncSearch: migrateAsyncSearch(src.AsyncSearch), - Admin: migrateAdmin(src.Admin, v1Cache), + SeqAPI: m.migrateSeqAPI(src.SeqAPI, v1Cache), + ErrorGroups: m.migrateErrorGroups(src.ErrorGroups), + AsyncSearch: m.migrateAsyncSearch(src.AsyncSearch), + Admin: m.migrateAdmin(src.Admin, v1Cache), } if src.MassExport != nil { - dst.MassExport = migrateMassExport(src.MassExport, cfg) + dst.MassExport = m.migrateMassExport(src.MassExport, cfg) } return dst } -func migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { - cacheID, redisID := defaultCacheIDs(v1Cache) +func (m *v1tov2) migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { + cacheID, redisID := m.defaultCacheIDs(v1Cache) dst := v2.SeqAPI{ - Envs: migrateSeqAPIEnvs(src.Envs, cacheID, redisID), + Envs: m.migrateSeqAPIEnvs(src.Envs, cacheID, redisID), DefaultEnv: src.DefaultEnv, Options: v2.SeqAPIOptions{ Caches: v2.SeqAPICaches{ - CacheID: cacheID, - RedisID: redisID, + Events: v2.SeqAPICache{ + ID: cacheID, + TTL: src.EventsCacheTTL, + }, + LogsLifespan: v2.SeqAPICache{ + ID: redisID, + TTL: src.LogsLifespanCacheTTL, + }, + Fields: v2.SeqAPICache{ + ID: cacheID, + TTL: src.FieldsCacheTTL, + }, }, }, } @@ -296,13 +317,13 @@ func migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { } if src.SeqAPIOptions != nil { - dst.Options = migrateSeqAPIOptions(src.SeqAPIOptions, cacheID, redisID) + dst.Options = m.migrateSeqAPIOptions(src.SeqAPIOptions, cacheID, redisID) } return dst } -func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv, cacheID, redisID string) map[string]v2.SeqAPIEnv { +func (m *v1tov2) migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv, cacheID, redisID string) map[string]v2.SeqAPIEnv { if len(envs) == 0 { return nil } @@ -313,7 +334,7 @@ func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv, cacheID, redisID string) ma SeqDBID: cfg.SeqDB, } if cfg.Options != nil { - envOptions := migrateSeqAPIOptions(cfg.Options, cacheID, redisID) + envOptions := m.migrateSeqAPIOptions(cfg.Options, cacheID, redisID) env.Options = &envOptions } @@ -323,7 +344,7 @@ func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv, cacheID, redisID string) ma return dst } -func migrateSeqAPIOptions(options *v1.SeqAPIOptions, cacheID, redisID string) v2.SeqAPIOptions { +func (m *v1tov2) migrateSeqAPIOptions(options *v1.SeqAPIOptions, cacheID, redisID string) v2.SeqAPIOptions { return v2.SeqAPIOptions{ Limits: v2.SeqAPILimits{ MaxSearchLimit: options.MaxSearchLimit, @@ -335,22 +356,27 @@ func migrateSeqAPIOptions(options *v1.SeqAPIOptions, cacheID, redisID string) v2 MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, }, - Masking: migrateMasking(options.Masking), - PinnedFields: migrateFields(options.PinnedFields), - SystemFields: migrateFields(options.SystemFields), + Masking: m.migrateMasking(options.Masking), + PinnedFields: m.migrateFields(options.PinnedFields), + SystemFields: m.migrateFields(options.SystemFields), Caches: v2.SeqAPICaches{ - CacheID: cacheID, - RedisID: redisID, - TTL: v2.SeqAPICachesTTL{ - Events: options.EventsCacheTTL, - LogsLifespan: options.LogsLifespanCacheTTL, - Fields: options.FieldsCacheTTL, + Events: v2.SeqAPICache{ + ID: cacheID, + TTL: options.EventsCacheTTL, + }, + LogsLifespan: v2.SeqAPICache{ + ID: redisID, + TTL: options.LogsLifespanCacheTTL, + }, + Fields: v2.SeqAPICache{ + ID: cacheID, + TTL: options.FieldsCacheTTL, }, }, } } -func migrateFields(fs []v1.Field) []v2.Field { +func (m *v1tov2) migrateFields(fs []v1.Field) []v2.Field { if fs == nil { return nil } @@ -366,19 +392,19 @@ func migrateFields(fs []v1.Field) []v2.Field { return dst } -func migrateMasking(src *v1.Masking) *v2.Masking { +func (m *v1tov2) migrateMasking(src *v1.Masking) *v2.Masking { if src == nil { return nil } return &v2.Masking{ - Masks: migrateMasks(src.Masks), + Masks: m.migrateMasks(src.Masks), ProcessFields: src.ProcessFields, IgnoreFields: src.IgnoreFields, } } -func migrateMasks(ms []v1.Mask) []v2.Mask { +func (m *v1tov2) migrateMasks(ms []v1.Mask) []v2.Mask { if ms == nil { return nil } @@ -393,14 +419,14 @@ func migrateMasks(ms []v1.Mask) []v2.Mask { ReplaceWord: m.ReplaceWord, ProcessFields: m.ProcessFields, IgnoreFields: m.IgnoreFields, - FieldFilters: migrateFieldFilters(m.FieldFilters), + FieldFilters: m.migrateFieldFilters(m.FieldFilters), } } return dst } -func migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { +func (m *v1tov2) migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { if src == nil { return nil } @@ -423,8 +449,8 @@ func migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { return dst } -func migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { - if isEgEmpty(eg) { +func (m *v1tov2) migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { + if m.isEgEmpty(eg) { return nil } @@ -439,7 +465,7 @@ func migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { } } -func migrateAsyncSearch(a v1.AsyncSearch) *v2.AsyncSearch { +func (m *v1tov2) migrateAsyncSearch(a v1.AsyncSearch) *v2.AsyncSearch { if len(a.AdminUsers) == 0 && a.ListQueryLengthLimit == 0 { return nil } @@ -451,7 +477,7 @@ func migrateAsyncSearch(a v1.AsyncSearch) *v2.AsyncSearch { } } -func migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { +func (m *v1tov2) migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { dst := &v2.MassExport{ SeqDBID: v2.DefaultSeqDBClientID, BatchSize: me.BatchSize, @@ -460,12 +486,12 @@ func migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { PartLength: me.PartLength, URLPrefix: me.URLPrefix, AllowedUsers: me.AllowedUsers, - FileStore: migrateFileStore(me.FileStore), - DownloadParams: migrateDownloadParams(me.SeqProxyDownloader), + FileStore: m.migrateFileStore(me.FileStore), + DownloadParams: m.migrateDownloadParams(me.SeqProxyDownloader), } if me.SessionStore != nil { - cfg.Cache.Redis = append(cfg.Cache.Redis, migrateRedis(&me.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) + cfg.Cache.Redis = append(cfg.Cache.Redis, m.migrateRedis(&me.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) dst.SessionStore = &v2.SessionStore{ RedisID: v2.DefaultMassExportRedisID, @@ -476,7 +502,7 @@ func migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { return dst } -func migrateFileStore(fs *v1.FileStore) *v2.FileStore { +func (m *v1tov2) migrateFileStore(fs *v1.FileStore) *v2.FileStore { if fs == nil { return nil } @@ -495,7 +521,7 @@ func migrateFileStore(fs *v1.FileStore) *v2.FileStore { return dst } -func migrateDownloadParams(src *v1.SeqProxyDownloader) *v2.DownloadParams { +func (m *v1tov2) migrateDownloadParams(src *v1.SeqProxyDownloader) *v2.DownloadParams { if src == nil { return nil } @@ -507,21 +533,23 @@ func migrateDownloadParams(src *v1.SeqProxyDownloader) *v2.DownloadParams { } } -func migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { +func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { if src == nil { return nil } - _, redisID := defaultCacheIDs(v1Cache) + _, redisID := m.defaultCacheIDs(v1Cache) return &v2.Admin{ - RedisID: redisID, SuperUsers: src.SuperUsers, - CacheTTL: src.CacheTTL, + Options: v2.HandlerCache{ + ID: redisID, + TTL: src.CacheTTL, + }, } } -func defaultCacheIDs(src v1.Cache) (cacheID, redisID string) { - hasInmem := !isInmemEmpty(src.Inmemory) +func (m *v1tov2) defaultCacheIDs(src v1.Cache) (cacheID, redisID string) { + hasInmem := !m.isInmemEmpty(src.Inmemory) hasRedis := src.Redis != nil switch { @@ -536,11 +564,11 @@ func defaultCacheIDs(src v1.Cache) (cacheID, redisID string) { } } -func isInmemEmpty(src v1.InmemoryCache) bool { +func (m *v1tov2) isInmemEmpty(src v1.InmemoryCache) bool { return src.BufferItems == 0 && src.MaxCost == 0 && src.NumCounters == 0 } -func isEgEmpty(src v1.ErrorGroups) bool { +func (m *v1tov2) isEgEmpty(src v1.ErrorGroups) bool { return len(src.LogTagsMapping.Env) == 0 && len(src.LogTagsMapping.Service) == 0 && len(src.LogTagsMapping.Release) == 0 && len(src.QueryFilter) == 0 } diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 1680952..df16c6e 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -34,9 +34,9 @@ import ( // skip_verify: // auth_urls: // tls: -// root_ca: // ca_cert: -// private_key: +// client_cert: +// client_key: // insecure: // allowed_clients: // jwt: @@ -71,10 +71,11 @@ import ( // initial_retry_backoff: // max_retry_backoff: // client_mode: -// grpc_keepalive_params: -// time: -// timeout: -// permit_without_stream: +// grpc_params: +// keep_alive: +// time: +// timeout: +// permit_without_stream: // download_params: // delay: // initial_retry_backoff: @@ -117,19 +118,22 @@ import ( // values: // process_fields: // ignore_fields: -// caches: -// cache_id: -// redis_id: -// ttl: -// events: -// logs_lifespan: -// fields: -// pinned_fields: -// - name: -// type: -// system_fields: -// - name: -// type: +// caches: +// events: +// id: +// ttl: +// logs_lifespan: +// id: +// ttl: +// fields: +// id: +// ttl: +// pinned_fields: +// - name: +// type: +// system_fields: +// - name: +// type: // envs: // : // seq_db_id: @@ -181,11 +185,12 @@ import ( // seq_db_id: // options: # same as root options // default_env: -// admin: -// options: -// redis_id: -// super_users: -// cache_ttl: +// admin: +// options: +// super_users: +// cache: +// id: +// ttl: // envs: // : // options: # same as root options @@ -237,9 +242,9 @@ type CORS struct { } type TLS struct { - RootCA string `yaml:"root_ca"` CACert string `yaml:"ca_cert"` - PrivateKey string `yaml:"private_key"` + ClientCert string `yaml:"client_cert"` + ClientKey string `yaml:"client_key"` Insecure bool `yaml:"insecure"` } @@ -426,7 +431,7 @@ type RateLimitersEnv struct { Options ApiToRateLimiters `yaml:"options"` } -type GRPCKeepaliveParams struct { +type KeepaliveParams struct { // After a duration of this time if the client doesn't see any activity it // pings the server to see if the transport is still alive. // If set below 10s, a minimum value of 10s will be used instead. @@ -441,17 +446,21 @@ type GRPCKeepaliveParams struct { PermitWithoutStream bool `yaml:"permit_without_stream"` } +type GRPCParams struct { + Keepalive *KeepaliveParams `yaml:"keep_alive"` +} + type SeqDBClient struct { - ID string `yaml:"id"` - Timeout time.Duration `yaml:"timeout"` - AvgDocSize int `yaml:"avg_doc_size"` - Addrs []string `yaml:"addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ClientMode string `yaml:"client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` - DownloadParams *DownloadParams `yaml:"download_params"` + ID string `yaml:"id"` + Timeout time.Duration `yaml:"timeout"` + AvgDocSize int `yaml:"avg_doc_size"` + Addrs []string `yaml:"addrs"` + RequestRetries int `yaml:"request_retries"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + ClientMode string `yaml:"client_mode"` + GRPCParams *GRPCParams `yaml:"grpc_params"` + DownloadParams *DownloadParams `yaml:"download_params"` } type CHClient struct { @@ -503,9 +512,8 @@ type AdminEnv struct { } type AdminOptions struct { - RedisID string `yaml:"redis_id"` - SuperUsers []string `yaml:"super_users"` - CacheTTL time.Duration `yaml:"cache_ttl"` + SuperUsers []string `yaml:"super_users"` + Cache HandlerCache `yaml:"cache"` } type Field struct { @@ -545,15 +553,14 @@ type SeqAPILimits struct { } type SeqAPICaches struct { - CacheID string `yaml:"cache_id"` // redis + inmem - RedisID string `yaml:"redis_id"` // redis only - TTL SeqAPICachesTTL `yaml:"ttl"` + Events HandlerCache `yaml:"events"` + LogsLifespan HandlerCache `yaml:"logs_lifespan"` + Fields HandlerCache `yaml:"fields"` } -type SeqAPICachesTTL struct { - Events time.Duration `yaml:"events"` - LogsLifespan time.Duration `yaml:"logs_lifespan"` - Fields time.Duration `yaml:"fields"` +type HandlerCache struct { + ID string `yaml:"id"` + TTL time.Duration `yaml:"ttl"` } type Masking struct { diff --git a/internal/app/config/v2/normalize.go b/internal/app/config/v2/normalize.go index 8cd58fb..fe1f46a 100644 --- a/internal/app/config/v2/normalize.go +++ b/internal/app/config/v2/normalize.go @@ -52,9 +52,11 @@ const ( defaultClickHouseDialTimeout = 5 * time.Second defaultClickHouseReadTimeout = 30 * time.Second + + defaultUsePreparedStatements = true ) -func Normalize(cfg *Config) error { +func (cfg *Config) Validate() error { if len(cfg.Clients.SeqDB) == 0 { return fmt.Errorf("clients.seq_db must contain at least one client") } @@ -77,9 +79,11 @@ func Normalize(cfg *Config) error { return fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, SeqDBClientModeGRPC) } - if c.GRPCKeepaliveParams != nil { - c.GRPCKeepaliveParams.Time = max(c.GRPCKeepaliveParams.Time, minGRPCKeepaliveTime) - c.GRPCKeepaliveParams.Timeout = max(c.GRPCKeepaliveParams.Timeout, minGRPCKeepaliveTimeout) + if c.GRPCParams != nil { + if c.GRPCParams.Keepalive != nil { + c.GRPCParams.Keepalive.Time = max(c.GRPCParams.Keepalive.Time, minGRPCKeepaliveTime) + c.GRPCParams.Keepalive.Timeout = max(c.GRPCParams.Keepalive.Timeout, minGRPCKeepaliveTimeout) + } } } @@ -146,8 +150,8 @@ func Normalize(cfg *Config) error { } if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { - cfg.DB.UsePreparedStatements = new(bool) - *cfg.DB.UsePreparedStatements = true + v := defaultUsePreparedStatements + cfg.DB.UsePreparedStatements = &v } if cfg.Server.Auth != nil && cfg.Server.Auth.Options.OIDC != nil { @@ -288,17 +292,19 @@ func Normalize(cfg *Config) error { if cfg.Handlers.Admin != nil { admin := cfg.Handlers.Admin - if admin.Options.CacheTTL <= 0 { - admin.Options.CacheTTL = defaultAdminCacheTTL - } - - if admin.Options.RedisID != "" { - redisCfg := cfg.Cache.RedisByID(admin.Options.RedisID) - if redisCfg == nil { - return fmt.Errorf("unknown handlers.admin.redis_id %q", admin.Options.RedisID) + if admin.Options.Cache != nil { + if admin.Options.Cache.TTL <= 0 { + admin.Options.Cache.TTL = defaultAdminCacheTTL } - if redisCfg.WithInmemID != "" { - return fmt.Errorf("handlers.admin.redis_id %q: with_inmem_id is not allowed", admin.Options.RedisID) + + if admin.Options.Cache.ID != "" { + redisCfg := cfg.Cache.RedisByID(admin.Options.Cache.ID) + if redisCfg == nil { + return fmt.Errorf("unknown handlers.admin.redis_id %q", admin.Options.Cache.ID) + } + if redisCfg.WithInmemID != "" { + return fmt.Errorf("handlers.admin.redis_id %q: with_inmem_id is not allowed", admin.Options.Cache.ID) + } } } } @@ -331,10 +337,10 @@ func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { if options.Limits.SeqCLIMaxSearchLimit <= 0 { options.Limits.SeqCLIMaxSearchLimit = 1000 // Сюда тоже или в них не было небходимости вообще, чтобы они были по нулям } - if options.Caches.TTL.Events <= 0 { - options.Caches.TTL.Events = defaultEventsCacheTTL + if options.Caches.Events.TTL <= 0 { + options.Caches.Events.TTL = defaultEventsCacheTTL } - if options.Caches.TTL.LogsLifespan <= 0 { - options.Caches.TTL.LogsLifespan = defaultLogsLifespanCacheTTL + if options.Caches.LogsLifespan.TTL <= 0 { + options.Caches.LogsLifespan.TTL = defaultLogsLifespanCacheTTL } } From 8ae7ed26c6f2cdc2be64fab08530cacc903cb889 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 7 Oct 2026 13:34:11 +0300 Subject: [PATCH 16/29] some refactor --- internal/app/config/loader/loader.go | 58 +-------------- internal/app/config/loader/merge.go | 72 +++++++++++++++++++ internal/app/config/migrate/v1_v2.go | 67 +++++++++-------- internal/app/config/migrate/v1_v2_test.go | 1 + internal/app/config/v2/config.go | 68 ++++++++---------- .../config/v2/{normalize.go => validate.go} | 36 ++++++---- internal/pkg/client/seqdb/grpc_client.go | 8 +-- 7 files changed, 165 insertions(+), 145 deletions(-) create mode 100644 internal/app/config/loader/merge.go create mode 100644 internal/app/config/migrate/v1_v2_test.go rename internal/app/config/v2/{normalize.go => validate.go} (92%) diff --git a/internal/app/config/loader/loader.go b/internal/app/config/loader/loader.go index af41a6c..34f6971 100644 --- a/internal/app/config/loader/loader.go +++ b/internal/app/config/loader/loader.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "fmt" - "maps" "os" "gopkg.in/yaml.v3" @@ -52,7 +51,7 @@ func FromFile(cfgPath string) (v2.Config, error) { return v2.Config{}, fmt.Errorf("unsupported config version: %d", version) } - cfgBytes, err = mergeHandlersEnvOptions(cfgBytes) + cfgBytes, err = mergeEnvOptions(cfgBytes) if err != nil { return v2.Config{}, fmt.Errorf("merge env options: %w", err) } @@ -126,58 +125,3 @@ func encode(cfg v2.Config) ([]byte, error) { } return buf.Bytes(), nil } - -func mergeHandlersEnvOptions(cfgBytes []byte) ([]byte, error) { - var root map[string]any - if err := yaml.Unmarshal(cfgBytes, &root); err != nil { - return nil, fmt.Errorf("parse yaml for merge: %w", err) - } - - handlers, _ := root["handlers"].(map[string]any) - for _, h := range handlers { - handler, _ := h.(map[string]any) - if handler == nil { - continue - } - - rootOpts := handler["options"].(map[string]any) - envs := handlers["envs"].(map[string]any) - if rootOpts == nil || len(envs) == 0 { - continue - } - - for _, e := range envs { - env, _ := e.(map[string]any) - if env == nil { - continue - } - - envOpts, _ := env["options"].(map[string]any) - env["options"] = mergeYAMLs(rootOpts, envOpts) - } - } - - out, err := yaml.Marshal(root) - if err != nil { - return nil, fmt.Errorf("marshal merged yaml: %w", err) - } - return out, nil -} - -func mergeYAMLs(a, b map[string]any) map[string]any { - merged := make(map[string]any) - maps.Copy(merged, a) - - for k, v := range b { - if existingValue, exists := merged[k]; exists { - if existingMap, ok := existingValue.(map[string]any); ok { - if newMap, ok := v.(map[string]any); ok { - merged[k] = mergeYAMLs(existingMap, newMap) - continue - } - } - } - merged[k] = v - } - return merged -} diff --git a/internal/app/config/loader/merge.go b/internal/app/config/loader/merge.go new file mode 100644 index 0000000..c05b866 --- /dev/null +++ b/internal/app/config/loader/merge.go @@ -0,0 +1,72 @@ +package loader + +import ( + "fmt" + "maps" + + "gopkg.in/yaml.v3" +) + +func mergeEnvOptions(cfgBytes []byte) ([]byte, error) { + var root map[string]any + if err := yaml.Unmarshal(cfgBytes, &root); err != nil { + return nil, fmt.Errorf("parse yaml for merge: %w", err) + } + + merge := func(root map[string]any) { + rootOpts, _ := root["options"].(map[string]any) + envs, _ := root["envs"].(map[string]any) + if rootOpts == nil || len(envs) == 0 { + return + } + + for _, e := range envs { + env, _ := e.(map[string]any) + if env == nil { + continue + } + + envOpts, _ := env["options"].(map[string]any) + env["options"] = mergeYAMLs(rootOpts, envOpts) + } + } + + handlers, _ := root["handlers"].(map[string]any) + for _, h := range handlers { + if handler, _ := h.(map[string]any); handler != nil { + merge(handler) + } + } + + server, _ := root["server"].(map[string]any) + if auth, _ := server["auth"].(map[string]any); auth != nil { + merge(auth) + } + if rl, _ := server["rate_limiters"].(map[string]any); rl != nil { + merge(rl) + } + + out, err := yaml.Marshal(root) + if err != nil { + return nil, fmt.Errorf("marshal merged yaml: %w", err) + } + return out, nil +} + +func mergeYAMLs(rootOpts, envOpts map[string]any) map[string]any { + merged := make(map[string]any) + maps.Copy(merged, rootOpts) + + for k, v := range envOpts { + if existingValue, exists := merged[k]; exists { + if existingMap, ok := existingValue.(map[string]any); ok { + if newMap, ok := v.(map[string]any); ok { + merged[k] = mergeYAMLs(existingMap, newMap) + continue + } + } + } + merged[k] = v + } + return merged +} diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index c76ab92..e843ea7 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -42,7 +42,9 @@ func (m *v1tov2) migrateServer(src *v1.Server) *v2.Server { Debug: v2.Debug{ Addr: src.DebugAddr, }, - RateLimiters: m.migrateApiRateLimiters(src.RateLimiters), + RateLimiters: &v2.RateLimiters{ + Options: m.migrateApiRateLimiters(src.RateLimiters), + }, } if src.OIDC != nil { @@ -58,14 +60,14 @@ func (m *v1tov2) migrateServer(src *v1.Server) *v2.Server { oidc.CacheID = cacheID } - dst.Auth = &v2.Auth{OIDC: oidc} + dst.Auth = &v2.Auth{Options: v2.AuthOptions{OIDC: oidc}} } if src.JWTSecretKey != "" { if dst.Auth == nil { dst.Auth = &v2.Auth{} } - dst.Auth.JWT = &v2.JWT{SecretKey: src.JWTSecretKey} + dst.Auth.Options.JWT = &v2.JWT{SecretKey: src.JWTSecretKey} } return dst @@ -296,15 +298,15 @@ func (m *v1tov2) migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { DefaultEnv: src.DefaultEnv, Options: v2.SeqAPIOptions{ Caches: v2.SeqAPICaches{ - Events: v2.SeqAPICache{ + Events: v2.HandlerCache{ ID: cacheID, TTL: src.EventsCacheTTL, }, - LogsLifespan: v2.SeqAPICache{ + LogsLifespan: v2.HandlerCache{ ID: redisID, TTL: src.LogsLifespanCacheTTL, }, - Fields: v2.SeqAPICache{ + Fields: v2.HandlerCache{ ID: cacheID, TTL: src.FieldsCacheTTL, }, @@ -360,15 +362,15 @@ func (m *v1tov2) migrateSeqAPIOptions(options *v1.SeqAPIOptions, cacheID, redisI PinnedFields: m.migrateFields(options.PinnedFields), SystemFields: m.migrateFields(options.SystemFields), Caches: v2.SeqAPICaches{ - Events: v2.SeqAPICache{ + Events: v2.HandlerCache{ ID: cacheID, TTL: options.EventsCacheTTL, }, - LogsLifespan: v2.SeqAPICache{ + LogsLifespan: v2.HandlerCache{ ID: redisID, TTL: options.LogsLifespanCacheTTL, }, - Fields: v2.SeqAPICache{ + Fields: v2.HandlerCache{ ID: cacheID, TTL: options.FieldsCacheTTL, }, @@ -411,15 +413,15 @@ func (m *v1tov2) migrateMasks(ms []v1.Mask) []v2.Mask { dst := make([]v2.Mask, len(ms)) for i := range ms { - m := &ms[i] + mask := &ms[i] dst[i] = v2.Mask{ - Re: m.Re, - Groups: m.Groups, - Mode: m.Mode, - ReplaceWord: m.ReplaceWord, - ProcessFields: m.ProcessFields, - IgnoreFields: m.IgnoreFields, - FieldFilters: m.migrateFieldFilters(m.FieldFilters), + Re: mask.Re, + Groups: mask.Groups, + Mode: mask.Mode, + ReplaceWord: mask.ReplaceWord, + ProcessFields: mask.ProcessFields, + IgnoreFields: mask.IgnoreFields, + FieldFilters: m.migrateFieldFilters(mask.FieldFilters), } } @@ -434,7 +436,6 @@ func (m *v1tov2) migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet dst := &v2.FieldFilterSet{ Condition: src.Condition, } - if src.Filters != nil { dst.Filters = make([]v2.FieldFilter, len(src.Filters)) for i, f := range src.Filters { @@ -456,12 +457,14 @@ func (m *v1tov2) migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { return &v2.ErrorGroups{ CHID: v2.DefaultCHClientID, - LogTagsMapping: v2.LogTagsMapping{ - Env: eg.LogTagsMapping.Env, - Service: eg.LogTagsMapping.Service, - Release: eg.LogTagsMapping.Release, + Options: v2.ErrorGroupsOptions{ + LogTagsMapping: v2.LogTagsMapping{ + Env: eg.LogTagsMapping.Env, + Service: eg.LogTagsMapping.Service, + Release: eg.LogTagsMapping.Release, + }, + QueryFilter: eg.QueryFilter, }, - QueryFilter: eg.QueryFilter, } } @@ -471,9 +474,11 @@ func (m *v1tov2) migrateAsyncSearch(a v1.AsyncSearch) *v2.AsyncSearch { } return &v2.AsyncSearch{ - SeqDBID: v2.DefaultSeqDBClientID, - AdminUsers: a.AdminUsers, - ListQueryLengthLimit: a.ListQueryLengthLimit, + SeqDBID: v2.DefaultSeqDBClientID, + Options: v2.AsyncSearchOptions{ + AdminUsers: a.AdminUsers, + ListQueryLengthLimit: a.ListQueryLengthLimit, + }, } } @@ -540,10 +545,12 @@ func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { _, redisID := m.defaultCacheIDs(v1Cache) return &v2.Admin{ - SuperUsers: src.SuperUsers, - Options: v2.HandlerCache{ - ID: redisID, - TTL: src.CacheTTL, + Options: v2.AdminOptions{ + Cache: &v2.HandlerCache{ + ID: redisID, + TTL: src.CacheTTL, + }, + SuperUsers: src.SuperUsers, }, } } diff --git a/internal/app/config/migrate/v1_v2_test.go b/internal/app/config/migrate/v1_v2_test.go new file mode 100644 index 0000000..278a35f --- /dev/null +++ b/internal/app/config/migrate/v1_v2_test.go @@ -0,0 +1 @@ +package migrate diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index df16c6e..8da85fc 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "slices" "time" ) @@ -223,12 +224,12 @@ import ( // max_retry_backoff: type Config struct { - Version int `yaml:"version"` - Server *Server `yaml:"server"` - Clients *Clients `yaml:"clients"` - Handlers *Handlers `yaml:"handlers"` - DB *DB `yaml:"db"` - Cache *Cache `yaml:"cache"` + Version int `yaml:"version"` + Server Server `yaml:"server"` + Clients Clients `yaml:"clients"` + Handlers Handlers `yaml:"handlers"` + Cache Cache `yaml:"cache"` + DB *DB `yaml:"db"` } type CORS struct { @@ -236,8 +237,8 @@ type CORS struct { AllowedMethods []string `yaml:"allowed_methods"` AllowedHeaders []string `yaml:"allowed_headers"` ExposedHeaders []string `yaml:"exposed_headers"` - AllowCredentials bool `yaml:"allow_credentials"` MaxAge int `yaml:"max_age"` + AllowCredentials bool `yaml:"allow_credentials"` OptionsPassthrough bool `yaml:"options_passthrough"` } @@ -316,31 +317,25 @@ type Cache struct { } func (c *Cache) InmemByID(id string) *InmemoryCache { - if c == nil { + i := slices.IndexFunc(c.Inmemory, func(inm InmemoryCache) bool { + return inm.ID == id + }) + if i == -1 { return nil } - for i := range c.Inmemory { - if c.Inmemory[i].ID == id { - return &c.Inmemory[i] - } - } - - return nil + return &c.Inmemory[i] } func (c *Cache) RedisByID(id string) *Redis { - if c == nil { + i := slices.IndexFunc(c.Redis, func(r Redis) bool { + return r.ID == id + }) + if i == -1 { return nil } - for i := range c.Redis { - if c.Redis[i].ID == id { - return &c.Redis[i] - } - } - - return nil + return &c.Redis[i] } type S3 struct { @@ -415,11 +410,11 @@ type AuthOptions struct { } type Server struct { - HTTP HTTP `yaml:"http"` - GRPC GRPC `yaml:"grpc"` - Debug Debug `yaml:"debug"` - Auth *Auth `yaml:"auth"` - RateLimiters RateLimiters `yaml:"rate_limiters"` + HTTP HTTP `yaml:"http"` + GRPC GRPC `yaml:"grpc"` + Debug Debug `yaml:"debug"` + Auth *Auth `yaml:"auth"` + RateLimiters *RateLimiters `yaml:"rate_limiters"` } type RateLimiters struct { @@ -480,17 +475,14 @@ type Clients struct { } func (c *Clients) ClickHouseByID(id string) *CHClient { - if c == nil { + i := slices.IndexFunc(c.ClickHouse, func(ch CHClient) bool { + return ch.ID == id + }) + if i == -1 { return nil } - for i := range c.ClickHouse { - if c.ClickHouse[i].ID == id { - return &c.ClickHouse[i] - } - } - - return nil + return &c.ClickHouse[i] } type Handlers struct { @@ -512,8 +504,8 @@ type AdminEnv struct { } type AdminOptions struct { - SuperUsers []string `yaml:"super_users"` - Cache HandlerCache `yaml:"cache"` + SuperUsers []string `yaml:"super_users"` + Cache *HandlerCache `yaml:"cache"` } type Field struct { diff --git a/internal/app/config/v2/normalize.go b/internal/app/config/v2/validate.go similarity index 92% rename from internal/app/config/v2/normalize.go rename to internal/app/config/v2/validate.go index fe1f46a..8ad63e1 100644 --- a/internal/app/config/v2/normalize.go +++ b/internal/app/config/v2/validate.go @@ -28,8 +28,9 @@ const ( FieldFilterModePrefix = "prefix" FieldFilterModeSuffix = "suffix" - minGRPCKeepaliveTime = 10 * time.Second - minGRPCKeepaliveTimeout = 1 * time.Second + defaultGRPCConnectionTimeout = 120 * time.Second + minGRPCKeepaliveTime = 10 * time.Second + minGRPCKeepaliveTimeout = 1 * time.Second defaultAsyncSearchListQueryLengthLimit = 1000 @@ -57,6 +58,10 @@ const ( ) func (cfg *Config) Validate() error { + if cfg.Server.GRPC.ConnectionTimeout <= 0 { + cfg.Server.GRPC.ConnectionTimeout = defaultGRPCConnectionTimeout + } + if len(cfg.Clients.SeqDB) == 0 { return fmt.Errorf("clients.seq_db must contain at least one client") } @@ -73,6 +78,12 @@ func (cfg *Config) Validate() error { seqDBIDs[c.ID] = struct{}{} + if len(c.Addrs) == 0 { + return fmt.Errorf("clients.seq_db[%q].addrs cannot be empty", c.ID) + } + if c.Timeout <= 0 { + return fmt.Errorf("clients.seq_db[%q].timeout must be > 0", c.ID) + } if c.ClientMode == "" { c.ClientMode = SeqDBClientModeGRPC } else if c.ClientMode != SeqDBClientModeGRPC { @@ -292,19 +303,16 @@ func (cfg *Config) Validate() error { if cfg.Handlers.Admin != nil { admin := cfg.Handlers.Admin - if admin.Options.Cache != nil { - if admin.Options.Cache.TTL <= 0 { - admin.Options.Cache.TTL = defaultAdminCacheTTL + if admin.Options.Cache.TTL <= 0 { + admin.Options.Cache.TTL = defaultAdminCacheTTL + } + if admin.Options.Cache.ID != "" { + redisCfg := cfg.Cache.RedisByID(admin.Options.Cache.ID) + if redisCfg == nil { + return fmt.Errorf("unknown handlers.admin.redis_id %q", admin.Options.Cache.ID) } - - if admin.Options.Cache.ID != "" { - redisCfg := cfg.Cache.RedisByID(admin.Options.Cache.ID) - if redisCfg == nil { - return fmt.Errorf("unknown handlers.admin.redis_id %q", admin.Options.Cache.ID) - } - if redisCfg.WithInmemID != "" { - return fmt.Errorf("handlers.admin.redis_id %q: with_inmem_id is not allowed", admin.Options.Cache.ID) - } + if redisCfg.WithInmemID != "" { + return fmt.Errorf("handlers.admin.redis_id %q: with_inmem_id is not allowed", admin.Options.Cache.ID) } } } diff --git a/internal/pkg/client/seqdb/grpc_client.go b/internal/pkg/client/seqdb/grpc_client.go index 2cdddee..86c0772 100644 --- a/internal/pkg/client/seqdb/grpc_client.go +++ b/internal/pkg/client/seqdb/grpc_client.go @@ -31,8 +31,8 @@ type grpcSearchResp interface { func processClientRequestUnaryInterceptor( ctx context.Context, method string, - req interface{}, - reply interface{}, + req any, + reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption, @@ -130,10 +130,6 @@ type GRPCClient struct { } func NewGRPCClient(ctx context.Context, params ClientParams) (*GRPCClient, error) { - if len(params.Addrs) == 0 { - panic("addrs is empty") - } - clients := make([]seqproxyapi.SeqProxyApiClient, 0, len(params.Addrs)) unaryInterceptors := []grpc.UnaryClientInterceptor{ processClientRequestUnaryInterceptor, From abe2ed3e416592e44cd1a545485a52d0b21e1198 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 7 Oct 2026 14:12:49 +0300 Subject: [PATCH 17/29] merge main --- internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go | 2 +- .../pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go | 2 +- pkg/admin/v1/admin.pb.go | 2 +- pkg/admin/v1/admin_grpc.pb.go | 2 +- pkg/dashboards/v1/dashboards.pb.go | 2 +- pkg/dashboards/v1/dashboards_grpc.pb.go | 2 +- pkg/errorgroups/v1/errorgroups.pb.go | 2 +- pkg/errorgroups/v1/errorgroups_grpc.pb.go | 2 +- pkg/massexport/v1/massexport.pb.go | 2 +- pkg/massexport/v1/massexport_grpc.pb.go | 2 +- pkg/seqapi/v1/seq_api.pb.go | 2 +- pkg/seqapi/v1/seq_api_grpc.pb.go | 2 +- pkg/userprofile/v1/userprofile.pb.go | 2 +- pkg/userprofile/v1/userprofile_grpc.pb.go | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go index 557cf70..d7979b4 100644 --- a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go +++ b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.36.2 +// protoc v7.36.0 // source: v1/seq_proxy_api.proto package seqproxyapi diff --git a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go index e683f13..0723644 100644 --- a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go +++ b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.36.2 +// - protoc v7.36.0 // source: v1/seq_proxy_api.proto package seqproxyapi diff --git a/pkg/admin/v1/admin.pb.go b/pkg/admin/v1/admin.pb.go index a8008aa..75e308f 100644 --- a/pkg/admin/v1/admin.pb.go +++ b/pkg/admin/v1/admin.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.35.1 +// protoc v7.36.0 // source: admin/v1/admin.proto package admin diff --git a/pkg/admin/v1/admin_grpc.pb.go b/pkg/admin/v1/admin_grpc.pb.go index 28e44d7..2fc96ed 100644 --- a/pkg/admin/v1/admin_grpc.pb.go +++ b/pkg/admin/v1/admin_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.35.1 +// - protoc v7.36.0 // source: admin/v1/admin.proto package admin diff --git a/pkg/dashboards/v1/dashboards.pb.go b/pkg/dashboards/v1/dashboards.pb.go index d11a515..42d26bd 100644 --- a/pkg/dashboards/v1/dashboards.pb.go +++ b/pkg/dashboards/v1/dashboards.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.35.1 +// protoc v7.36.0 // source: dashboards/v1/dashboards.proto package dashboards diff --git a/pkg/dashboards/v1/dashboards_grpc.pb.go b/pkg/dashboards/v1/dashboards_grpc.pb.go index 9559155..c7a0d0d 100644 --- a/pkg/dashboards/v1/dashboards_grpc.pb.go +++ b/pkg/dashboards/v1/dashboards_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.35.1 +// - protoc v7.36.0 // source: dashboards/v1/dashboards.proto package dashboards diff --git a/pkg/errorgroups/v1/errorgroups.pb.go b/pkg/errorgroups/v1/errorgroups.pb.go index aed151f..aa9bfff 100644 --- a/pkg/errorgroups/v1/errorgroups.pb.go +++ b/pkg/errorgroups/v1/errorgroups.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.35.1 +// protoc v7.36.0 // source: errorgroups/v1/errorgroups.proto package errorgroups diff --git a/pkg/errorgroups/v1/errorgroups_grpc.pb.go b/pkg/errorgroups/v1/errorgroups_grpc.pb.go index 84bd4bf..2944ad3 100644 --- a/pkg/errorgroups/v1/errorgroups_grpc.pb.go +++ b/pkg/errorgroups/v1/errorgroups_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.35.1 +// - protoc v7.36.0 // source: errorgroups/v1/errorgroups.proto package errorgroups diff --git a/pkg/massexport/v1/massexport.pb.go b/pkg/massexport/v1/massexport.pb.go index 3c5f3e9..efdbf27 100644 --- a/pkg/massexport/v1/massexport.pb.go +++ b/pkg/massexport/v1/massexport.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.35.1 +// protoc v7.36.0 // source: massexport/v1/massexport.proto package massexport diff --git a/pkg/massexport/v1/massexport_grpc.pb.go b/pkg/massexport/v1/massexport_grpc.pb.go index a78e188..ed37283 100644 --- a/pkg/massexport/v1/massexport_grpc.pb.go +++ b/pkg/massexport/v1/massexport_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.35.1 +// - protoc v7.36.0 // source: massexport/v1/massexport.proto package massexport diff --git a/pkg/seqapi/v1/seq_api.pb.go b/pkg/seqapi/v1/seq_api.pb.go index 5d48a2e..6c146b2 100644 --- a/pkg/seqapi/v1/seq_api.pb.go +++ b/pkg/seqapi/v1/seq_api.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.36.2 +// protoc v7.36.0 // source: seqapi/v1/seq_api.proto package seqapi diff --git a/pkg/seqapi/v1/seq_api_grpc.pb.go b/pkg/seqapi/v1/seq_api_grpc.pb.go index e51df99..393bb69 100644 --- a/pkg/seqapi/v1/seq_api_grpc.pb.go +++ b/pkg/seqapi/v1/seq_api_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.36.2 +// - protoc v7.36.0 // source: seqapi/v1/seq_api.proto package seqapi diff --git a/pkg/userprofile/v1/userprofile.pb.go b/pkg/userprofile/v1/userprofile.pb.go index e0f53f0..02c11f0 100644 --- a/pkg/userprofile/v1/userprofile.pb.go +++ b/pkg/userprofile/v1/userprofile.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.35.1 +// protoc v7.36.0 // source: userprofile/v1/userprofile.proto package userprofile diff --git a/pkg/userprofile/v1/userprofile_grpc.pb.go b/pkg/userprofile/v1/userprofile_grpc.pb.go index 93ce310..31c1ffe 100644 --- a/pkg/userprofile/v1/userprofile_grpc.pb.go +++ b/pkg/userprofile/v1/userprofile_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.35.1 +// - protoc v7.36.0 // source: userprofile/v1/userprofile.proto package userprofile From 99f86e8ec7e3a43d7b0fb09e363f28e8cd0b1f56 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Tue, 13 Oct 2026 01:25:19 +0300 Subject: [PATCH 18/29] update config.server --- cmd/seq-ui/main.go | 23 +- internal/api/seqapi/v1/api_error/error.go | 6 +- internal/api/seqapi/v1/grpc/aggregation.go | 3 +- internal/api/seqapi/v1/grpc/api.go | 12 +- internal/api/seqapi/v1/grpc/cluster_status.go | 3 +- internal/api/seqapi/v1/grpc/events.go | 3 +- internal/api/seqapi/v1/grpc/fields.go | 3 +- internal/api/seqapi/v1/grpc/histogram.go | 3 +- internal/api/seqapi/v1/grpc/limits.go | 3 +- internal/api/seqapi/v1/grpc/logs_lifespan.go | 3 +- internal/api/seqapi/v1/grpc/search.go | 3 +- internal/api/seqapi/v1/http/aggregation.go | 9 +- .../api/seqapi/v1/http/aggregation_test.go | 24 +- internal/api/seqapi/v1/http/aggregation_ts.go | 7 +- .../api/seqapi/v1/http/aggregation_ts_test.go | 40 +- internal/api/seqapi/v1/http/api.go | 117 +-- internal/api/seqapi/v1/http/cluster_status.go | 3 +- internal/api/seqapi/v1/http/events.go | 16 +- internal/api/seqapi/v1/http/export.go | 6 +- .../api/seqapi/v1/http/export_async_search.go | 6 +- .../v1/http/export_async_search_test.go | 40 +- internal/api/seqapi/v1/http/export_test.go | 34 +- internal/api/seqapi/v1/http/fields.go | 98 +- internal/api/seqapi/v1/http/fields_test.go | 16 +- internal/api/seqapi/v1/http/get_envs_test.go | 72 +- internal/api/seqapi/v1/http/histogram.go | 3 +- internal/api/seqapi/v1/http/limits.go | 23 +- internal/api/seqapi/v1/http/limits_test.go | 12 +- internal/api/seqapi/v1/http/logs_lifespan.go | 19 +- .../api/seqapi/v1/http/logs_lifespan_test.go | 8 +- internal/api/seqapi/v1/http/mw.go | 23 - internal/api/seqapi/v1/http/search.go | 23 +- internal/api/seqapi/v1/http/search_test.go | 4 +- .../api/seqapi/v1/http/start_async_search.go | 4 +- internal/api/seqapi/v1/http/test_data.go | 2 +- internal/api/seqapi/v1/seqapi.go | 5 +- internal/app/auth/oidc.go | 12 +- internal/app/config/migrate/v1_v2.go | 880 +++++++++--------- internal/app/config/migrate/v1_v2_test.go | 2 + internal/app/config/v1/config.go | 2 +- internal/app/config/v2/config.go | 274 +++--- internal/app/config/v2/validate.go | 663 +++++++------ internal/app/mw/auth.go | 31 +- internal/app/mw/env.go | 69 ++ internal/app/mw/grpc_mw.go | 47 +- internal/app/mw/http_mw.go | 40 +- internal/app/mw/ratelimiter.go | 36 +- internal/app/server/init.go | 137 ++- internal/app/server/run.go | 110 ++- internal/app/server/server.go | 15 +- internal/app/server/tech.go | 8 +- internal/app/types/ctx.go | 15 + internal/pkg/cache/ctor.go | 4 +- internal/pkg/cache/inmemory.go | 2 +- .../pkg/service/massexport/filestore/s3.go | 2 +- 55 files changed, 1698 insertions(+), 1330 deletions(-) delete mode 100644 internal/api/seqapi/v1/http/mw.go create mode 100644 internal/app/mw/env.go diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index d1edfb7..bcb66e2 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -94,24 +94,19 @@ func run(ctx context.Context) { } logger.Info("initializing caches") - caches, err := cache.FromConfig(ctx, cfg.Cache) + caches, err := cache.FromConfig(ctx, &cfg.Cache) if err != nil { logger.Fatal("failed to init caches", zap.Error(err)) } - var oidcCache cache.Cache - if cfg.Server.Auth != nil && cfg.Server.Auth.OIDC != nil { - oidcCache = caches[cfg.Server.Auth.OIDC.CacheID] - } - registrar := initApp(ctx, cfg, caches) - serv, err := server.New(ctx, cfg.Server, registrar, oidcCache) + serv, err := server.New(ctx, &cfg, registrar, caches) if err != nil { logger.Fatal("app init error", zap.Error(err)) } - // Run launches both grpc and http servers. On successful + // Run launches configured grpc/http servers and debug server. On successful // http.ErrServerClosed is returned because of http.Server.Serve. if err = serv.Run(ctx); !errors.Is(err, http.ErrServerClosed) { logger.Fatal("app run", zap.Error(err)) @@ -181,9 +176,15 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac } } - seqApiCache := caches[cfg.Handlers.SeqAPI.CacheID] - seqApiRedisCache := caches[cfg.Handlers.SeqAPI.RedisID] - seqApiV1 := seqapi_v1.New(&cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesSvc) + seqAPICaches := cfg.Handlers.SeqAPI.Options.Caches + seqApiV1 := seqapi_v1.New( + &cfg.Handlers.SeqAPI, + seqDBClients, + caches[seqAPICaches.Events.ID], + caches[seqAPICaches.LogsLifespan.ID], + caches[seqAPICaches.Fields.ID], + asyncSearchesSvc, + ) var errorGroupsV1 *errorgroups_v1.ErrorGroups if cfg.Handlers.ErrorGroups != nil { diff --git a/internal/api/seqapi/v1/api_error/error.go b/internal/api/seqapi/v1/api_error/error.go index 915c567..c203b8c 100644 --- a/internal/api/seqapi/v1/api_error/error.go +++ b/internal/api/seqapi/v1/api_error/error.go @@ -12,21 +12,21 @@ var ( ErrOffsetConflict = errors.New(`only one of "offset" and "offset_id" must be provided`) ) -func CheckSearchLimit(limit, maximum int32) error { +func CheckSearchLimit(limit, maximum int) error { if limit == 0 || limit > maximum { return fmt.Errorf("'limit' must be greater than 0 and less than %v", maximum) } return nil } -func CheckSearchOffsetLimit(offset, maximum int32) error { +func CheckSearchOffsetLimit(offset, maximum int) error { if offset > maximum { return ErrQueryTooHeavy } return nil } -func CheckForOffsetConflict(offset int32, offsetID string) error { +func CheckForOffsetConflict(offset int, offsetID string) error { if offset != 0 && offsetID != "" { return ErrOffsetConflict } diff --git a/internal/api/seqapi/v1/grpc/aggregation.go b/internal/api/seqapi/v1/grpc/aggregation.go index 9603b88..3ae1f21 100644 --- a/internal/api/seqapi/v1/grpc/aggregation.go +++ b/internal/api/seqapi/v1/grpc/aggregation.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/seqapi/v1/aggregation_ts" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -19,7 +20,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ defer span.End() aggregations, _ := json.Marshal(req.Aggregations) - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) attributes := []attribute.KeyValue{ { diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index d425633..642e9e0 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -1,14 +1,12 @@ package grpc import ( - "context" "fmt" "sort" "strconv" "time" "go.uber.org/zap" - "google.golang.org/grpc/metadata" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" @@ -52,6 +50,7 @@ func New( seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, + fieldsCache cache.Cache, asyncSearches asyncsearches.Service, ) *API { globalParams := apiGlobalParams{ @@ -158,15 +157,6 @@ func parseEnvs(cfg *config.SeqAPI) *seqapi.GetEnvsResponse { return &seqapi.GetEnvsResponse{Envs: envs} } -func (a *API) GetEnvFromContext(ctx context.Context) string { - md, _ := metadata.FromIncomingContext(ctx) - envValues := md.Get("env") - if len(envValues) == 0 { - return "" - } - return envValues[0] -} - func (a *API) GetParams(env string) (apiParams, error) { if len(a.config.Envs) == 0 { return a.paramsByEnv[""], nil diff --git a/internal/api/seqapi/v1/grpc/cluster_status.go b/internal/api/seqapi/v1/grpc/cluster_status.go index 0b09e6e..06f6b0e 100644 --- a/internal/api/seqapi/v1/grpc/cluster_status.go +++ b/internal/api/seqapi/v1/grpc/cluster_status.go @@ -6,6 +6,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -14,7 +15,7 @@ func (a *API) Status(ctx context.Context, req *seqapi.StatusRequest) (*seqapi.St ctx, span := tracing.StartSpan(ctx, "seqapi_v1_status") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/internal/api/seqapi/v1/grpc/events.go b/internal/api/seqapi/v1/grpc/events.go index aea707b..bda573b 100644 --- a/internal/api/seqapi/v1/grpc/events.go +++ b/internal/api/seqapi/v1/grpc/events.go @@ -9,6 +9,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" @@ -18,7 +19,7 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_event") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) attributes := []attribute.KeyValue{ { diff --git a/internal/api/seqapi/v1/grpc/fields.go b/internal/api/seqapi/v1/grpc/fields.go index f07109f..84d11b8 100644 --- a/internal/api/seqapi/v1/grpc/fields.go +++ b/internal/api/seqapi/v1/grpc/fields.go @@ -7,6 +7,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" @@ -16,7 +17,7 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_fields") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/internal/api/seqapi/v1/grpc/histogram.go b/internal/api/seqapi/v1/grpc/histogram.go index 0d82303..63d6805 100644 --- a/internal/api/seqapi/v1/grpc/histogram.go +++ b/internal/api/seqapi/v1/grpc/histogram.go @@ -7,6 +7,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -15,7 +16,7 @@ func (a *API) GetHistogram(ctx context.Context, req *seqapi.GetHistogramRequest) ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_histogram") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) attributes := []attribute.KeyValue{ { diff --git a/internal/api/seqapi/v1/grpc/limits.go b/internal/api/seqapi/v1/grpc/limits.go index 2021bcb..0a5b702 100644 --- a/internal/api/seqapi/v1/grpc/limits.go +++ b/internal/api/seqapi/v1/grpc/limits.go @@ -6,11 +6,12 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) func (a *API) GetLimits(ctx context.Context, _ *seqapi.GetLimitsRequest) (*seqapi.GetLimitsResponse, error) { - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan.go b/internal/api/seqapi/v1/grpc/logs_lifespan.go index ed3dec8..0bb265e 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan.go @@ -14,6 +14,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/grpcutil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/lifespan" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -24,7 +25,7 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_logs_lifespan") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/internal/api/seqapi/v1/grpc/search.go b/internal/api/seqapi/v1/grpc/search.go index 02149cf..7046cc9 100644 --- a/internal/api/seqapi/v1/grpc/search.go +++ b/internal/api/seqapi/v1/grpc/search.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/seqapi/v1/aggregation_ts" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -18,7 +19,7 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se ctx, span := tracing.StartSpan(ctx, "seqapi_v1_search") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) spanAttributes := []attribute.KeyValue{ { diff --git a/internal/api/seqapi/v1/http/aggregation.go b/internal/api/seqapi/v1/http/aggregation.go index e3c8dae..3a5dab8 100644 --- a/internal/api/seqapi/v1/http/aggregation.go +++ b/internal/api/seqapi/v1/http/aggregation.go @@ -11,6 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -40,7 +41,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { } aggsRaw, _ := json.Marshal(httpReq.Aggregations) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) attributes := []attribute.KeyValue{ { @@ -81,7 +82,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { return } - if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { wr.Error(err, http.StatusBadRequest) return } @@ -94,7 +95,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { getAggResp := getAggregationResponseFromProto(resp) - if a.globalParams.masker != nil { + if params.masker != nil { buf := make([]string, 0) for i, agg := range getAggResp.Aggregations { buf = buf[:0] @@ -108,7 +109,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { field = aggReq.GroupBy } - buf = a.globalParams.masker.MaskAgg(field, buf) + buf = params.masker.MaskAgg(field, buf) for j, key := range buf { getAggResp.Aggregations[i].Buckets[j].Key = key diff --git a/internal/api/seqapi/v1/http/aggregation_test.go b/internal/api/seqapi/v1/http/aggregation_test.go index 76978d9..150063e 100644 --- a/internal/api/seqapi/v1/http/aggregation_test.go +++ b/internal/api/seqapi/v1/http/aggregation_test.go @@ -64,7 +64,9 @@ func TestServeGetAggregation(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -107,7 +109,9 @@ func TestServeGetAggregation(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -169,7 +173,9 @@ func TestServeGetAggregation(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -204,7 +210,9 @@ func TestServeGetAggregation(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -219,7 +227,9 @@ func TestServeGetAggregation(t *testing.T) { wantErr: true, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 2, + }, }, }, }, @@ -243,7 +253,9 @@ func TestServeGetAggregation(t *testing.T) { wantErr: true, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, diff --git a/internal/api/seqapi/v1/http/aggregation_ts.go b/internal/api/seqapi/v1/http/aggregation_ts.go index f4d5ac5..130d9e7 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts.go +++ b/internal/api/seqapi/v1/http/aggregation_ts.go @@ -13,6 +13,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/aggregation_ts" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -34,7 +35,7 @@ func (a *API) serveGetAggregationTs(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) var httpReq getAggregationTsRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -75,13 +76,13 @@ func (a *API) serveGetAggregationTs(w http.ResponseWriter, r *http.Request) { return } - if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { wr.Error(err, http.StatusBadRequest) return } for _, agg := range httpReq.Aggregations { if err := api_error.CheckAggregationTsInterval(agg.Interval, httpReq.From, httpReq.To, - params.options.MaxBucketsPerAggregationTs, + params.options.Limits.MaxBucketsPerAggregationTs, ); err != nil { wr.Error(err, http.StatusBadRequest) return diff --git a/internal/api/seqapi/v1/http/aggregation_ts_test.go b/internal/api/seqapi/v1/http/aggregation_ts_test.go index d6a8752..852b3e8 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts_test.go +++ b/internal/api/seqapi/v1/http/aggregation_ts_test.go @@ -107,8 +107,10 @@ func TestServeGetAggregationTs(t *testing.T) { wantStatus: http.StatusOK, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -163,8 +165,10 @@ func TestServeGetAggregationTs(t *testing.T) { wantStatus: http.StatusOK, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -222,8 +226,10 @@ func TestServeGetAggregationTs(t *testing.T) { wantStatus: http.StatusOK, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -248,8 +254,10 @@ func TestServeGetAggregationTs(t *testing.T) { wantStatus: http.StatusOK, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -264,8 +272,10 @@ func TestServeGetAggregationTs(t *testing.T) { wantStatus: http.StatusBadRequest, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, - MaxBucketsPerAggregationTs: 100, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 2, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -279,8 +289,10 @@ func TestServeGetAggregationTs(t *testing.T) { wantStatus: http.StatusBadRequest, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 1, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 1, + }, }, }, }, @@ -298,7 +310,9 @@ func TestServeGetAggregationTs(t *testing.T) { wantStatus: http.StatusInternalServerError, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index 09d0133..93e2cf4 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -22,26 +22,23 @@ import ( ) type apiParams struct { - client seqdb.Client - options *config.SeqAPIOptions - exportLimiter *tokenlimiter.Limiter -} - -type apiGlobalParams struct { - fieldsCache *fieldsCache + client seqdb.Client + options *config.SeqAPIOptions masker *mask.Masker pinnedFields fields systemFields fields - eventsCacheTTL time.Duration - logsLifespanCacheTTL time.Duration + exportLimiter *tokenlimiter.Limiter + eventCacheKeyPrefix string + fieldsCacheKey string + logsLifespanCacheKey string } type API struct { config *config.SeqAPI - globalParams apiGlobalParams paramsByEnv map[string]apiParams inmemWithRedisCache cache.Cache redisCache cache.Cache + fieldsCache *fieldsCache nowFn func() time.Time asyncSearches asyncsearches.Service envsResponse getEnvsResponse @@ -52,68 +49,60 @@ func New( seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, + fieldsCache cache.Cache, asyncSearches asyncsearches.Service, ) *API { - globalParams := apiGlobalParams{ - pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), - systemFields: parseFields(cfg.GlobalOptions.SystemFields), - eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, - logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, - } - - if cfg.GlobalOptions.FieldsCacheTTL > 0 { - globalParams.fieldsCache = newFieldsCache(cfg.GlobalOptions.FieldsCacheTTL) - } - - globalMasker, err := mask.New(cfg.GlobalOptions.Masking) - if err != nil { - logger.Fatal("failed to init masking", zap.Error(err)) - } - - globalParams.masker = globalMasker - globalExportLimiter := tokenlimiter.New(cfg.Options.MaxParallelExportRequests) - paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { for envName, envConfig := range cfg.Envs { - paramsByEnv[envName] = apiParams{ - client: seqDBСlients[envConfig.SeqDBID], - options: envConfig.Options, - exportLimiter: tokenlimiter.New(envConfig.Options.MaxParallelExportRequests), - } + paramsByEnv[envName] = newAPIParams(envName, seqDBСlients[envConfig.SeqDBID], envConfig.Options) } } else { - paramsByEnv[""] = apiParams{ - client: seqDBСlients[cfg.SeqDBID], - options: &cfg.Options, - exportLimiter: globalExportLimiter, - } - } - - // for export - if globalParams.masker != nil { - for _, param := range paramsByEnv { - param.client.WithMasking(globalParams.masker) - } + paramsByEnv[""] = newAPIParams("", seqDBСlients[cfg.SeqDBID], &cfg.Options) } return &API{ config: cfg, - globalParams: globalParams, paramsByEnv: paramsByEnv, inmemWithRedisCache: inmemWithRedisCache, redisCache: redisCache, + fieldsCache: newFieldsCache(fieldsCache), nowFn: time.Now, asyncSearches: asyncSearches, envsResponse: parseEnvs(cfg), } } +func newAPIParams(env string, client seqdb.Client, options *config.SeqAPIOptions) apiParams { + masker, err := mask.New(options.Masking) + if err != nil { + logger.Fatal("failed to init masking", zap.Error(err)) + } + + // for export + if masker != nil { + client.WithMasking(masker) + } + + cacheKeyPrefix := "seqapi_" + env + "_" + + return apiParams{ + client: client, + options: options, + masker: masker, + pinnedFields: parseFields(options.PinnedFields), + systemFields: parseFields(options.SystemFields), + exportLimiter: tokenlimiter.New(options.Limits.MaxParallelExportRequests), + eventCacheKeyPrefix: cacheKeyPrefix + "event_", + fieldsCacheKey: cacheKeyPrefix + "fields", + logsLifespanCacheKey: cacheKeyPrefix + "logs_lifespan", + } +} + func (a *API) Router() chi.Router { mux := chi.NewMux() - mux.Use(a.envInterceptor) - + // seqapi mux.Post("/aggregation", a.serveGetAggregation) mux.Post("/aggregation_ts", a.serveGetAggregationTs) mux.Get("/events/{id}", a.serveGetEvent) @@ -252,26 +241,6 @@ func apiErrorFromProto(proto *seqapi.Error) apiError { } } -type fieldsCache struct { - ttl time.Duration - - ts time.Time - rawFields []byte -} - -func newFieldsCache(ttl time.Duration) *fieldsCache { - return &fieldsCache{ttl: ttl} -} - -func (c *fieldsCache) getFields() ([]byte, bool, bool) { - return c.rawFields, !c.ts.IsZero(), time.Since(c.ts) < c.ttl -} - -func (c *fieldsCache) setFields(rawFields []byte) { - c.rawFields = rawFields - c.ts = time.Now() -} - type asyncSearchStatus string // @name seqapi.v1.AsyncSearchStatus const ( @@ -331,10 +300,10 @@ func checkLimitOffset(limit, offset int32) error { func createEnvInfo(envName string, opts *config.SeqAPIOptions) envInfo { return envInfo{ Env: envName, - MaxSearchLimit: uint32(opts.MaxSearchLimit), - MaxExportLimit: uint32(opts.MaxExportLimit), - MaxParallelExportRequests: uint32(opts.MaxParallelExportRequests), - MaxAggregationsPerRequest: uint32(opts.MaxAggregationsPerRequest), - SeqCliMaxSearchLimit: uint32(opts.SeqCLIMaxSearchLimit), + MaxSearchLimit: uint32(opts.Limits.MaxSearchLimit), + MaxExportLimit: uint32(opts.Limits.MaxExportLimit), + MaxParallelExportRequests: uint32(opts.Limits.MaxParallelExportRequests), + MaxAggregationsPerRequest: uint32(opts.Limits.MaxAggregationsPerRequest), + SeqCliMaxSearchLimit: uint32(opts.Limits.SeqCLIMaxSearchLimit), } } diff --git a/internal/api/seqapi/v1/http/cluster_status.go b/internal/api/seqapi/v1/http/cluster_status.go index 8f8ec13..4f1ca1b 100644 --- a/internal/api/seqapi/v1/http/cluster_status.go +++ b/internal/api/seqapi/v1/http/cluster_status.go @@ -7,6 +7,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -25,7 +26,7 @@ func (a *API) serveStatus(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { wr.Error(err, http.StatusBadRequest) diff --git a/internal/api/seqapi/v1/http/events.go b/internal/api/seqapi/v1/http/events.go index 0f45dde..26fffc0 100644 --- a/internal/api/seqapi/v1/http/events.go +++ b/internal/api/seqapi/v1/http/events.go @@ -10,6 +10,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" @@ -33,7 +34,7 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) attributes := []attribute.KeyValue{ { @@ -54,11 +55,12 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { return } - if cached, err := a.inmemWithRedisCache.Get(ctx, id); err == nil { + cacheKey := params.eventCacheKeyPrefix + id + if cached, err := a.inmemWithRedisCache.Get(ctx, cacheKey); err == nil { e := &seqapi.Event{} if err = proto.Unmarshal([]byte(cached), e); err == nil { - if a.globalParams.masker != nil { - a.globalParams.masker.Mask(e.Data) + if params.masker != nil { + params.masker.Mask(e.Data) } wr.WriteJson(getEventResponseFromProto(&seqapi.GetEventResponse{Event: e})) return @@ -74,12 +76,12 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { return } - if a.globalParams.masker != nil && resp.Event != nil { - a.globalParams.masker.Mask(resp.Event.Data) + if params.masker != nil && resp.Event != nil { + params.masker.Mask(resp.Event.Data) } if data, err := proto.Marshal(resp.Event); err == nil { - _ = a.inmemWithRedisCache.SetWithTTL(ctx, id, string(data), a.globalParams.eventsCacheTTL) + _ = a.inmemWithRedisCache.SetWithTTL(ctx, cacheKey, string(data), params.options.Caches.Events.TTL) } else { logger.Error("failed to marshal event proto for caching", zap.String("id", id), zap.Error(err)) } diff --git a/internal/api/seqapi/v1/http/export.go b/internal/api/seqapi/v1/http/export.go index 379f17b..1b929af 100644 --- a/internal/api/seqapi/v1/http/export.go +++ b/internal/api/seqapi/v1/http/export.go @@ -33,7 +33,7 @@ func (a *API) serveExport(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) userStr := "_" if userName, err := types.GetUserKey(ctx); err == nil { @@ -104,9 +104,9 @@ func (a *API) serveExport(w http.ResponseWriter, r *http.Request) { } defer params.exportLimiter.Fill(userStr) - if httpReq.Limit > params.options.MaxExportLimit { + if httpReq.Limit > params.options.Limits.MaxExportLimit { wr.Error(fmt.Errorf("too many events are requested: count=%d, max=%d", - httpReq.Limit, params.options.MaxExportLimit), + httpReq.Limit, params.options.Limits.MaxExportLimit), http.StatusBadRequest) return } diff --git a/internal/api/seqapi/v1/http/export_async_search.go b/internal/api/seqapi/v1/http/export_async_search.go index 493b1b2..1afc559 100644 --- a/internal/api/seqapi/v1/http/export_async_search.go +++ b/internal/api/seqapi/v1/http/export_async_search.go @@ -36,7 +36,7 @@ func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_export_async_search_http") defer span.End() - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) userStr := "_" if userName, err := types.GetUserKey(ctx); err == nil { @@ -105,9 +105,9 @@ func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { } defer params.exportLimiter.Fill(userStr) - if httpReq.Limit > params.options.MaxExportLimit { + if httpReq.Limit > params.options.Limits.MaxExportLimit { wr.Error(fmt.Errorf("too many events are requested: count=%d, max=%d", - httpReq.Limit, params.options.MaxExportLimit), + httpReq.Limit, params.options.Limits.MaxExportLimit), http.StatusBadRequest) return } diff --git a/internal/api/seqapi/v1/http/export_async_search_test.go b/internal/api/seqapi/v1/http/export_async_search_test.go index b640fee..7e6150b 100644 --- a/internal/api/seqapi/v1/http/export_async_search_test.go +++ b/internal/api/seqapi/v1/http/export_async_search_test.go @@ -37,8 +37,10 @@ func TestServeExportAsyncSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, mockArgs: &mockArgs{ @@ -61,8 +63,10 @@ func TestServeExportAsyncSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, mockArgs: &mockArgs{ @@ -84,7 +88,9 @@ func TestServeExportAsyncSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxParallelExportRequests: 0, + Limits: config.SeqAPILimits{ + MaxParallelExportRequests: 0, + }, }, }, wantErr: true, @@ -98,8 +104,10 @@ func TestServeExportAsyncSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 5, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 5, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -114,8 +122,10 @@ func TestServeExportAsyncSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -129,8 +139,10 @@ func TestServeExportAsyncSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -144,8 +156,10 @@ func TestServeExportAsyncSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, diff --git a/internal/api/seqapi/v1/http/export_test.go b/internal/api/seqapi/v1/http/export_test.go index 2bedd30..a2f92e8 100644 --- a/internal/api/seqapi/v1/http/export_test.go +++ b/internal/api/seqapi/v1/http/export_test.go @@ -41,8 +41,10 @@ func TestServeExport(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, mockArgs: &mockArgs{ @@ -68,8 +70,10 @@ func TestServeExport(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, mockArgs: &mockArgs{ @@ -95,7 +99,9 @@ func TestServeExport(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxParallelExportRequests: 0, + Limits: config.SeqAPILimits{ + MaxParallelExportRequests: 0, + }, }, }, wantErr: true, @@ -111,8 +117,10 @@ func TestServeExport(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 5, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 5, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -129,8 +137,10 @@ func TestServeExport(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -146,8 +156,10 @@ func TestServeExport(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, diff --git a/internal/api/seqapi/v1/http/fields.go b/internal/api/seqapi/v1/http/fields.go index 7f4e8d1..6924938 100644 --- a/internal/api/seqapi/v1/http/fields.go +++ b/internal/api/seqapi/v1/http/fields.go @@ -1,18 +1,57 @@ package http import ( - "encoding/json" + "context" "net/http" + "time" "go.uber.org/zap" + "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) +type fieldsCache struct { + cache cache.Cache +} + +func newFieldsCache(c cache.Cache) *fieldsCache { + return &fieldsCache{cache: c} +} + +func (c *fieldsCache) get(ctx context.Context, key string) ([]*seqapi.Field, bool) { + data, err := c.cache.Get(ctx, key) + if err != nil { + return nil, false + } + + var resp seqapi.GetFieldsResponse + if err := proto.Unmarshal([]byte(data), &resp); err != nil { + logger.Error("can't unmarshal cached fields", zap.Error(err)) + return nil, false + } + + return resp.GetFields(), true +} + +func (c *fieldsCache) set(ctx context.Context, key string, fields []*seqapi.Field, ttl time.Duration) { + data, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: fields}) + if err != nil { + logger.Error("can't marshal fields for cache", zap.Error(err)) + return + } + + if err := c.cache.SetWithTTL(ctx, key, string(data), ttl); err != nil { + logger.Error("can't set fields to cache", zap.Error(err)) + } +} + // serveGetFields go doc. // // @Router /seqapi/v1/fields [get] @@ -27,14 +66,14 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { wr.Error(err, http.StatusBadRequest) return } - if a.globalParams.fieldsCache == nil { + if a.fieldsCache == nil { resp, err := params.client.GetFields(ctx, &seqapi.GetFieldsRequest{}) if err != nil { wr.Error(err, http.StatusInternalServerError) @@ -43,50 +82,37 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { res := getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), - SystemFields: a.globalParams.systemFields, - PinnedFields: a.globalParams.pinnedFields, + SystemFields: params.systemFields, + PinnedFields: params.pinnedFields, } wr.WriteJson(res) return } - rawFields, cached, isActual := a.globalParams.fieldsCache.getFields() - if cached && isActual { - _, _ = wr.Write(rawFields) + if cachedFields, ok := a.fieldsCache.get(ctx, params.fieldsCacheKey); ok { + res := getFieldsResponse{ + Fields: fieldsFromProto(cachedFields), + SystemFields: params.systemFields, + PinnedFields: params.pinnedFields, + } + wr.WriteJson(res) return } resp, err := params.client.GetFields(ctx, &seqapi.GetFieldsRequest{}) if err != nil { - if cached { - logger.Error("can't get fields; use cached fields", zap.Error(err)) - _, _ = wr.Write(rawFields) - return - } - wr.Error(err, http.StatusInternalServerError) return } + a.fieldsCache.set(ctx, params.fieldsCacheKey, resp.GetFields(), params.options.Caches.Fields.TTL) res := getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), - SystemFields: a.globalParams.systemFields, - PinnedFields: a.globalParams.pinnedFields, - } - resData, err := json.Marshal(res) - if err != nil { - if cached { - logger.Error("can't marshal fields; use cached fields", zap.Error(err)) - _, _ = wr.Write(rawFields) - return - } - - wr.Error(err, http.StatusInternalServerError) - return + SystemFields: params.systemFields, + PinnedFields: params.pinnedFields, } - a.globalParams.fieldsCache.setFields(resData) - _, _ = wr.Write(resData) + wr.WriteJson(res) } // serveGetPinnedFields go doc. @@ -98,8 +124,20 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { // @Success 200 {object} getFieldsResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" func (a *API) serveGetPinnedFields(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_pinned_fields") + defer span.End() + + wr := httputil.NewWriter(w) + + env := types.GetEnvKey(ctx) + params, err := a.GetEnvParams(env) + if err != nil { + wr.Error(err, http.StatusBadRequest) + return + } + httputil.NewWriter(w).WriteJson(getFieldsResponse{ - Fields: a.globalParams.pinnedFields, + Fields: params.pinnedFields, }) } diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index e8476b9..3c29e03 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -77,7 +77,7 @@ func TestServeGetFields(t *testing.T) { Fields: fAPI, }, }, - cfg: config.SeqAPIGlobalOptions{ + cfg: config.SeqAPIOptions{ SystemFields: []config.Field{ {Name: "field1", Type: "keyword"}, {Name: "field2", Type: "text"}, @@ -95,7 +95,11 @@ func TestServeGetFields(t *testing.T) { }, cacheData: marshal(getFieldsResponse{Fields: f}), cfg: config.SeqAPIOptions{ - FieldsCacheTTL: time.Hour, + Caches: config.SeqAPICaches{ + Fields: config.HandlerCache{ + TTL: time.Hour, + }, + }, }, }, { @@ -114,7 +118,7 @@ func TestServeGetFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: tt.cfg, + Options: tt.cfg, }, } @@ -129,8 +133,8 @@ func TestServeGetFields(t *testing.T) { Return(tt.mockArgs.resp, tt.mockArgs.err). Times(1) } - if tt.cacheData != nil && tt.cfg.FieldsCacheTTL > 0 { - api.params.fieldsCache.setFields(tt.cacheData) + if tt.cacheData != nil && tt.cfg.Caches.Fields.TTL > 0 { + api.fieldsCache.set(tt.cacheData) } httputil.DoTestHTTPEx(t, httputil.TestDataHTTPEx[struct{}, getFieldsResponse]{ @@ -172,7 +176,7 @@ func TestServeGetPinnedFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: config.SeqAPIGlobalOptions{ + Options: config.SeqAPIOptions{ PinnedFields: tt.fields, }, }, diff --git a/internal/api/seqapi/v1/http/get_envs_test.go b/internal/api/seqapi/v1/http/get_envs_test.go index 6e46ced..2e0fd59 100644 --- a/internal/api/seqapi/v1/http/get_envs_test.go +++ b/internal/api/seqapi/v1/http/get_envs_test.go @@ -19,11 +19,13 @@ func TestServeGetEnvs(t *testing.T) { name: "single_env", cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: getEnvsResponse{ @@ -46,51 +48,61 @@ func TestServeGetEnvs(t *testing.T) { "cluster-220": { SeqDBID: "pro-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 1000, - MaxExportLimit: 500, - MaxParallelExportRequests: 10, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 2000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 1000, + MaxExportLimit: 500, + MaxParallelExportRequests: 10, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 2000, + }, }, }, "cluster-10": { SeqDBID: "prod-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 1000, - MaxExportLimit: 500, - MaxParallelExportRequests: 10, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 2000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 1000, + MaxExportLimit: 500, + MaxParallelExportRequests: 10, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 2000, + }, }, }, "cluster-102": { SeqDBID: "staging-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, + }, }, }, "prod": { SeqDBID: "stag-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, + }, }, }, "wyanki": { SeqDBID: "sta-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, + }, }, }, }, diff --git a/internal/api/seqapi/v1/http/histogram.go b/internal/api/seqapi/v1/http/histogram.go index ba3bbc6..72717e7 100644 --- a/internal/api/seqapi/v1/http/histogram.go +++ b/internal/api/seqapi/v1/http/histogram.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -31,7 +32,7 @@ func (a *API) serveGetHistogram(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) var httpReq getHistogramRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { diff --git a/internal/api/seqapi/v1/http/limits.go b/internal/api/seqapi/v1/http/limits.go index b9f719a..f5bace7 100644 --- a/internal/api/seqapi/v1/http/limits.go +++ b/internal/api/seqapi/v1/http/limits.go @@ -4,6 +4,7 @@ import ( "net/http" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/tracing" ) @@ -20,7 +21,7 @@ func (a *API) serveGetLimits(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { @@ -29,18 +30,18 @@ func (a *API) serveGetLimits(w http.ResponseWriter, r *http.Request) { } httputil.NewWriter(w).WriteJson(getLimitsResponse{ - MaxSearchLimit: params.options.MaxSearchLimit, - MaxExportLimit: params.options.MaxExportLimit, - MaxParallelExportRequests: int32(params.options.MaxParallelExportRequests), - MaxAggregationsPerRequest: int32(params.options.MaxAggregationsPerRequest), - SeqCliMaxSearchLimit: int32(params.options.SeqCLIMaxSearchLimit), + MaxSearchLimit: params.options.Limits.MaxSearchLimit, + MaxExportLimit: params.options.Limits.MaxExportLimit, + MaxParallelExportRequests: params.options.Limits.MaxParallelExportRequests, + MaxAggregationsPerRequest: params.options.Limits.MaxAggregationsPerRequest, + SeqCliMaxSearchLimit: params.options.Limits.SeqCLIMaxSearchLimit, }) } type getLimitsResponse struct { - MaxSearchLimit int32 `json:"maxSearchLimit" format:"int32"` - MaxExportLimit int32 `json:"maxExportLimit" format:"int32"` - MaxParallelExportRequests int32 `json:"maxParallelExportRequests" format:"int32"` - MaxAggregationsPerRequest int32 `json:"maxAggregationsPerRequest" format:"int32"` - SeqCliMaxSearchLimit int32 `json:"seqCliMaxSearchLimit" format:"int32"` + MaxSearchLimit int `json:"maxSearchLimit"` + MaxExportLimit int `json:"maxExportLimit"` + MaxParallelExportRequests int `json:"maxParallelExportRequests"` + MaxAggregationsPerRequest int `json:"maxAggregationsPerRequest"` + SeqCliMaxSearchLimit int `json:"seqCliMaxSearchLimit"` } // @name seqapi.v1.GetLimitsResponse diff --git a/internal/api/seqapi/v1/http/limits_test.go b/internal/api/seqapi/v1/http/limits_test.go index eebb094..321b81e 100644 --- a/internal/api/seqapi/v1/http/limits_test.go +++ b/internal/api/seqapi/v1/http/limits_test.go @@ -22,11 +22,13 @@ func TestServeGetLimits(t *testing.T) { env: "default", cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: getLimitsResponse{ diff --git a/internal/api/seqapi/v1/http/logs_lifespan.go b/internal/api/seqapi/v1/http/logs_lifespan.go index 8d68811..0901bf0 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan.go +++ b/internal/api/seqapi/v1/http/logs_lifespan.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/lifespan" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -30,20 +31,19 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { wr.Error(err, http.StatusBadRequest) return } - cacheKey := "logs_lifespan" - if resStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { + if resStr, err := a.redisCache.Get(ctx, params.logsLifespanCacheKey); err == nil { res := 0 res, err = strconv.Atoi(resStr) if err == nil { logger.Debug("got logs lifespan from cache") - wr.WriteJson(getLogsLifespanResponse{Lifespan: int64(res)}) + wr.WriteJson(getLogsLifespanResponse{Lifespan: res}) return } @@ -65,15 +65,18 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { res := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) - err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(res), a.globalParams.logsLifespanCacheTTL) - if err != nil { + if err = a.redisCache.SetWithTTL( + ctx, + params.logsLifespanCacheKey, + strconv.Itoa(res), + params.options.Caches.LogsLifespan.TTL); err != nil { logger.Error("can't set logs lifespan to cache", zap.Error(err)) } logger.Debug("got logs lifespan from seq-proxy") - wr.WriteJson(getLogsLifespanResponse{Lifespan: int64(res)}) + wr.WriteJson(getLogsLifespanResponse{Lifespan: res}) } type getLogsLifespanResponse struct { - Lifespan int64 `json:"lifespan"` + Lifespan int `json:"lifespan"` } // @name seqapi.v1.GetLogsLifespanResponse diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index 3238563..1b9e300 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -103,8 +103,12 @@ func TestServeGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: config.SeqAPIGlobalOptions{ - LogsLifespanCacheTTL: cacheTTL, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + LogsLifespan: config.HandlerCache{ + TTL: cacheTTL, + }, + }, }, }, } diff --git a/internal/api/seqapi/v1/http/mw.go b/internal/api/seqapi/v1/http/mw.go deleted file mode 100644 index ef46456..0000000 --- a/internal/api/seqapi/v1/http/mw.go +++ /dev/null @@ -1,23 +0,0 @@ -package http - -import ( - "context" - "net/http" -) - -type envContextKey struct{} - -func getEnvFromContext(ctx context.Context) string { - if v := ctx.Value(envContextKey{}); v != nil { - return v.(string) - } - return "" -} - -func (a *API) envInterceptor(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - env := r.URL.Query().Get("env") - ctx := context.WithValue(r.Context(), envContextKey{}, env) - next.ServeHTTP(w, r.WithContext(ctx)) - }) -} diff --git a/internal/api/seqapi/v1/http/search.go b/internal/api/seqapi/v1/http/search.go index 6916242..ad20c6f 100644 --- a/internal/api/seqapi/v1/http/search.go +++ b/internal/api/seqapi/v1/http/search.go @@ -12,6 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -32,7 +33,7 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) var httpReq searchRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -109,11 +110,11 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { span.SetAttributes(spanAttributes...) - if err := api_error.CheckSearchLimit(httpReq.Limit, params.options.MaxSearchLimit); err != nil { + if err := api_error.CheckSearchLimit(httpReq.Limit, params.options.Limits.MaxSearchLimit); err != nil { wr.Error(err, http.StatusBadRequest) return } - if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { wr.Error(err, http.StatusBadRequest) return } @@ -121,7 +122,7 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { wr.Error(err, http.StatusBadRequest) return } - if err := api_error.CheckSearchOffsetLimit(httpReq.Offset, params.options.MaxSearchOffsetLimit); err != nil { + if err := api_error.CheckSearchOffsetLimit(httpReq.Offset, params.options.Limits.MaxSearchOffset); err != nil { wr.Error(err, http.StatusBadRequest) return } @@ -132,7 +133,7 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { return } - if resp.Total > params.options.MaxSearchTotalLimit { + if resp.Total > int64(params.options.Limits.MaxSearchTotal) { resp.Error = &seqapi.Error{ Code: seqapi.ErrorCode_ERROR_CODE_QUERY_TOO_HEAVY, Message: api_error.ErrQueryTooHeavy.Error(), @@ -140,9 +141,9 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { } searchResp := searchResponseFromProto(resp, httpReq.WithTotal) - if a.globalParams.masker != nil { + if params.masker != nil { for i := range searchResp.Events { - a.globalParams.masker.Mask(searchResp.Events[i].Data) + params.masker.Mask(searchResp.Events[i].Data) } } @@ -169,8 +170,8 @@ type searchRequest struct { Query string `json:"query"` From time.Time `json:"from" format:"date-time"` To time.Time `json:"to" format:"date-time"` - Limit int32 `json:"limit" format:"int32"` - Offset int32 `json:"offset" format:"int32"` + Limit int `json:"limit"` + Offset int `json:"offset"` WithTotal bool `json:"withTotal"` Aggregations aggregationQueries `json:"aggregations"` Histogram struct { @@ -186,8 +187,8 @@ func (r searchRequest) toProto() *seqapi.SearchRequest { Query: r.Query, From: timestamppb.New(r.From), To: timestamppb.New(r.To), - Limit: r.Limit, - Offset: r.Offset, + Limit: int32(r.Limit), + Offset: int32(r.Offset), OffsetId: r.OffsetID, WithTotal: r.WithTotal, Downsample: r.Downsample, diff --git a/internal/api/seqapi/v1/http/search_test.go b/internal/api/seqapi/v1/http/search_test.go index 945674f..b9b308d 100644 --- a/internal/api/seqapi/v1/http/search_test.go +++ b/internal/api/seqapi/v1/http/search_test.go @@ -52,7 +52,9 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index f2b548c..e124ff3 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -37,7 +37,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_start_async_search") defer span.End() - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { httputil.NewWriter(w).Error(err, http.StatusBadRequest) @@ -61,7 +61,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { continue } if err := api_error.CheckAggregationTsInterval(agg.Interval, httpReq.From, httpReq.To, - params.options.MaxBucketsPerAggregationTs, + params.options.Limits.MaxBucketsPerAggregationTs, ); err != nil { wr.Error(err, http.StatusBadRequest) return diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 6e0c0ff..4e727af 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -39,7 +39,7 @@ func setupTestAPI(data test.APITestData) *API { asyncSvc = data.Mocks.AsyncSearchesSvc } - return New(&data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) + return New(&data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) } func withQueryParamID(h http.HandlerFunc, id string) http.HandlerFunc { diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index 375e0e2..753f1c7 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -21,11 +21,12 @@ func New( seqDB map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, + fieldsCache cache.Cache, asyncSearches asyncsearches.Service, ) *SeqAPI { return &SeqAPI{ - grpcAPI: grpc_api.New(cfg, seqDB, inmemWithRedisCache, redisCache, asyncSearches), - httpAPI: http_api.New(cfg, seqDB, inmemWithRedisCache, redisCache, asyncSearches), + grpcAPI: grpc_api.New(cfg, seqDB, inmemWithRedisCache, redisCache, fieldsCache, asyncSearches), + httpAPI: http_api.New(cfg, seqDB, inmemWithRedisCache, redisCache, fieldsCache, asyncSearches), } } diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 790c2b8..002b3e7 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -170,15 +170,15 @@ func newHTTPContext(ctx context.Context, cfg *config.TLS) (context.Context, erro } b := tls.NewConfigBuilder() if cfg.Insecure { - b.SetInsecureSkipVerify(cfg.Insecure) - } else if !cfg.Insecure && (cfg.RootCA != "" || cfg.CACert != "" || cfg.PrivateKey != "") { - if cfg.RootCA != "" { - if err := b.AppendCARoot(cfg.RootCA); err != nil { + b.SetInsecureSkipVerify(true) + } else { + if cfg.CACert != "" { + if err := b.AppendCARoot(cfg.CACert); err != nil { return nil, fmt.Errorf("can't append CA root: %w", err) } } - if cfg.CACert != "" || cfg.PrivateKey != "" { - if err := b.AppendX509KeyPair(cfg.CACert, cfg.PrivateKey); err != nil { + if cfg.ClientCert != "" || cfg.ClientKey != "" { + if err := b.AppendX509KeyPair(cfg.ClientCert, cfg.ClientKey); err != nil { return nil, fmt.Errorf("can't append key pair: %w", err) } } diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index e843ea7..db612a9 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -16,35 +16,45 @@ func V1ToV2(src v1.Config) v2.Config { func (m *v1tov2) migrate() v2.Config { dst := v2.Config{Version: 2} - dst.Server = m.migrateServer(m.src.Server) - dst.Cache = m.migrateCache(m.src.Server) - dst.Clients = m.migrateClients(m.src) - dst.DB = m.migrateDB(m.src.Server) - dst.Handlers = m.migrateHandlers(m.src.Handlers, &dst, m.src.Server.Cache) + + // var v1Cache v1.Cache + // if m.src.Server != nil { + // v1Cache = m.src.Server.Cache + // } + // + // dst.Cache = m.migrateCache(v1Cache) + // dst.Clients = m.migrateClients(m.src) + // dst.DB = m.migrateDB(m.src.Server) + // dst.Handlers = m.migrateHandlers(m.src.Handlers, &dst, v1Cache) return dst } -func (m *v1tov2) migrateServer(src *v1.Server) *v2.Server { - dst := &v2.Server{ - HTTP: v2.HTTP{ +func (m *v1tov2) migrateServer(src *v1.Server) v2.Server { + if src == nil { + return v2.Server{} + } + + dst := v2.Server{ + HTTP: &v2.HTTP{ Addr: src.HTTPAddr, ReadTimeout: src.HTTPReadTimeout, ReadHeaderTimeout: src.HTTPReadHeaderTimeout, WriteTimeout: src.HTTPWriteTimeout, CORS: m.migrateCORS(src.CORS), }, - GRPC: v2.GRPC{ + GRPC: &v2.GRPC{ Addr: src.GRPCAddr, ConnectionTimeout: src.GRPCConnectionTimeout, }, Debug: v2.Debug{ - Addr: src.DebugAddr, - }, - RateLimiters: &v2.RateLimiters{ - Options: m.migrateApiRateLimiters(src.RateLimiters), + Addr: src.DebugAddr, + ReadTimeout: src.HTTPReadTimeout, + ReadHeaderTimeout: src.HTTPReadHeaderTimeout, + WriteTimeout: src.HTTPWriteTimeout, }, + RateLimiting: m.migrateRateLimiting(src.RateLimiters), } if src.OIDC != nil { @@ -103,30 +113,30 @@ func (m *v1tov2) migrateCORS(src *v1.CORS) *v2.CORS { } } -func (m *v1tov2) migrateApiRateLimiters(src v1.ApiToRateLimiters) v2.ApiToRateLimiters { - if src == nil { +func (m *v1tov2) migrateRateLimiting(src v1.ApiToRateLimiters) *v2.RateLimiting { + if len(src) == 0 { return nil } - dst := make(v2.ApiToRateLimiters, len(src)) + options := make(v2.ApiToRateLimiters, len(src)) for api, rl := range src { - dst[api] = v2.ApiRateLimiters{ + options[api] = v2.RateLimiters{ Default: m.migrateRateLimiter(rl.Default), SpecialUsers: m.migrateUserToRateLimiter(rl.SpecialUsers), } } - return dst + return &v2.RateLimiting{Options: options} } -func (m *v1tov2) migrateUserToRateLimiter(src v1.UserToRateLimiter) v2.UserToRateLimiter { +func (m *v1tov2) migrateUserToRateLimiter(src v1.UserToRateLimiter) map[string]v2.RateLimiter { if src == nil { return nil } - dst := make(v2.UserToRateLimiter, len(src)) - for k, rl := range src { - dst[k] = m.migrateRateLimiter(rl) + dst := make(map[string]v2.RateLimiter, len(src)) + for user, rl := range src { + dst[user] = m.migrateRateLimiter(rl) } return dst @@ -155,427 +165,413 @@ func (m *v1tov2) migrateGRPCParams(src *v1.GRPCKeepaliveParams) *v2.GRPCParams { } } -func (m *v1tov2) migrateClients(src v1.Config) *v2.Clients { - dst := &v2.Clients{} - - if src.Clients != nil { - if len(src.Clients.SeqDB) > 0 { - dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) - for i := range src.Clients.SeqDB { - dst.SeqDB = append(dst.SeqDB, m.migrateSeqDBClient(&src.Clients.SeqDB[i])) - } - } else { - dst.SeqDB = []v2.SeqDBClient{{ - ID: v2.DefaultSeqDBClientID, - Timeout: src.Clients.SeqDBTimeout, - AvgDocSize: src.Clients.SeqDBAvgDocSize, - Addrs: src.Clients.SeqDBAddrs, - RequestRetries: src.Clients.RequestRetries, - InitialRetryBackoff: src.Clients.InitialRetryBackoff, - MaxRetryBackoff: src.Clients.MaxRetryBackoff, - ClientMode: src.Clients.ProxyClientMode, - GRPCParams: m.migrateGRPCParams(src.Clients.GRPCKeepaliveParams), - }} - } - } - - if src.Server != nil && src.Server.CH != nil { - ch := src.Server.CH - dst.ClickHouse = []v2.CHClient{{ - ID: v2.DefaultCHClientID, - Addrs: ch.Addrs, - Database: ch.Database, - Username: ch.Username, - Password: ch.Password, - Sharded: ch.Sharded, - DialTimeout: ch.DialTimeout, - ReadTimeout: ch.ReadTimeout, - }} - } - - return dst -} - -func (m *v1tov2) migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { - return v2.SeqDBClient{ - ID: src.ID, - Timeout: src.Timeout, - AvgDocSize: src.AvgDocSize, - Addrs: src.Addrs, - RequestRetries: src.RequestRetries, - InitialRetryBackoff: src.InitialRetryBackoff, - MaxRetryBackoff: src.MaxRetryBackoff, - ClientMode: src.ClientMode, - GRPCParams: m.migrateGRPCParams(src.GRPCKeepaliveParams), - } -} - -func (m *v1tov2) migrateDB(src *v1.Server) *v2.DB { - if src == nil || src.DB == nil { - return nil - } - - return &v2.DB{ - Name: src.DB.Name, - Host: src.DB.Host, - Port: src.DB.Port, - Pass: src.DB.Pass, - User: src.DB.User, - RequestTimeout: src.DB.RequestTimeout, - ConnectionPoolCapacity: src.DB.ConnectionPoolCapacity, - UsePreparedStatements: src.DB.UsePreparedStatements, - } -} - -func (m *v1tov2) migrateCache(src *v1.Server) *v2.Cache { - if src == nil { - return &v2.Cache{} - } - - cache := &v2.Cache{} - - if !m.isInmemEmpty(src.Cache.Inmemory) { - cache.Inmemory = append(cache.Inmemory, v2.InmemoryCache{ - ID: v2.DefaultInmemCacheID, - NumCounters: src.Cache.Inmemory.NumCounters, - MaxCost: src.Cache.Inmemory.MaxCost, - BufferItems: src.Cache.Inmemory.BufferItems, - }) - } - - if src.Cache.Redis != nil { - if len(cache.Inmemory) > 0 { - cache.Redis = append(cache.Redis, - m.migrateRedis(src.Cache.Redis, v2.DefaultRedisID, v2.DefaultInmemCacheID), - m.migrateRedis(src.Cache.Redis, v2.DefaultRedis2ID, ""), - ) - } else { - cache.Redis = append(cache.Redis, m.migrateRedis(src.Cache.Redis, v2.DefaultRedisID, "")) - } - } - - return cache -} - -func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { - return v2.Redis{ - ID: id, - WithInmemID: withInmemID, - Addr: src.Addr, - Username: src.Username, - Password: src.Password, - Timeout: src.Timeout, - MaxRetries: src.MaxRetries, - MinRetryBackoff: src.MinRetryBackoff, - MaxRetryBackoff: src.MaxRetryBackoff, - KeyPrefix: src.KeyPrefix, - } -} - -func (m *v1tov2) migrateHandlers(src *v1.Handlers, cfg *v2.Config, v1Cache v1.Cache) *v2.Handlers { - if src == nil { - return &v2.Handlers{} - } - - dst := &v2.Handlers{ - SeqAPI: m.migrateSeqAPI(src.SeqAPI, v1Cache), - ErrorGroups: m.migrateErrorGroups(src.ErrorGroups), - AsyncSearch: m.migrateAsyncSearch(src.AsyncSearch), - Admin: m.migrateAdmin(src.Admin, v1Cache), - } - - if src.MassExport != nil { - dst.MassExport = m.migrateMassExport(src.MassExport, cfg) - } - - return dst -} - -func (m *v1tov2) migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { - cacheID, redisID := m.defaultCacheIDs(v1Cache) - dst := v2.SeqAPI{ - Envs: m.migrateSeqAPIEnvs(src.Envs, cacheID, redisID), - DefaultEnv: src.DefaultEnv, - Options: v2.SeqAPIOptions{ - Caches: v2.SeqAPICaches{ - Events: v2.HandlerCache{ - ID: cacheID, - TTL: src.EventsCacheTTL, - }, - LogsLifespan: v2.HandlerCache{ - ID: redisID, - TTL: src.LogsLifespanCacheTTL, - }, - Fields: v2.HandlerCache{ - ID: cacheID, - TTL: src.FieldsCacheTTL, - }, - }, - }, - } - - if len(src.Envs) == 0 { - dst.SeqDBID = v2.DefaultSeqDBClientID - } - - if src.SeqAPIOptions != nil { - dst.Options = m.migrateSeqAPIOptions(src.SeqAPIOptions, cacheID, redisID) - } - - return dst -} - -func (m *v1tov2) migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv, cacheID, redisID string) map[string]v2.SeqAPIEnv { - if len(envs) == 0 { - return nil - } - - dst := make(map[string]v2.SeqAPIEnv, len(envs)) - for name, cfg := range envs { - env := v2.SeqAPIEnv{ - SeqDBID: cfg.SeqDB, - } - if cfg.Options != nil { - envOptions := m.migrateSeqAPIOptions(cfg.Options, cacheID, redisID) - env.Options = &envOptions - } - - dst[name] = env - } - - return dst -} - -func (m *v1tov2) migrateSeqAPIOptions(options *v1.SeqAPIOptions, cacheID, redisID string) v2.SeqAPIOptions { - return v2.SeqAPIOptions{ - Limits: v2.SeqAPILimits{ - MaxSearchLimit: options.MaxSearchLimit, - MaxSearchTotal: options.MaxSearchTotalLimit, - MaxSearchOffset: options.MaxSearchOffsetLimit, - MaxExportLimit: options.MaxExportLimit, - SeqCLIMaxSearchLimit: options.SeqCLIMaxSearchLimit, - MaxParallelExportRequests: options.MaxParallelExportRequests, - MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, - MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, - }, - Masking: m.migrateMasking(options.Masking), - PinnedFields: m.migrateFields(options.PinnedFields), - SystemFields: m.migrateFields(options.SystemFields), - Caches: v2.SeqAPICaches{ - Events: v2.HandlerCache{ - ID: cacheID, - TTL: options.EventsCacheTTL, - }, - LogsLifespan: v2.HandlerCache{ - ID: redisID, - TTL: options.LogsLifespanCacheTTL, - }, - Fields: v2.HandlerCache{ - ID: cacheID, - TTL: options.FieldsCacheTTL, - }, - }, - } -} - -func (m *v1tov2) migrateFields(fs []v1.Field) []v2.Field { - if fs == nil { - return nil - } - - dst := make([]v2.Field, len(fs)) - for i, f := range fs { - dst[i] = v2.Field{ - Name: f.Name, - Type: f.Type, - } - } - - return dst -} - -func (m *v1tov2) migrateMasking(src *v1.Masking) *v2.Masking { - if src == nil { - return nil - } - - return &v2.Masking{ - Masks: m.migrateMasks(src.Masks), - ProcessFields: src.ProcessFields, - IgnoreFields: src.IgnoreFields, - } -} - -func (m *v1tov2) migrateMasks(ms []v1.Mask) []v2.Mask { - if ms == nil { - return nil - } - - dst := make([]v2.Mask, len(ms)) - for i := range ms { - mask := &ms[i] - dst[i] = v2.Mask{ - Re: mask.Re, - Groups: mask.Groups, - Mode: mask.Mode, - ReplaceWord: mask.ReplaceWord, - ProcessFields: mask.ProcessFields, - IgnoreFields: mask.IgnoreFields, - FieldFilters: m.migrateFieldFilters(mask.FieldFilters), - } - } - - return dst -} - -func (m *v1tov2) migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { - if src == nil { - return nil - } - - dst := &v2.FieldFilterSet{ - Condition: src.Condition, - } - if src.Filters != nil { - dst.Filters = make([]v2.FieldFilter, len(src.Filters)) - for i, f := range src.Filters { - dst.Filters[i] = v2.FieldFilter{ - Field: f.Field, - Mode: f.Mode, - Values: f.Values, - } - } - } - - return dst -} - -func (m *v1tov2) migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { - if m.isEgEmpty(eg) { - return nil - } - - return &v2.ErrorGroups{ - CHID: v2.DefaultCHClientID, - Options: v2.ErrorGroupsOptions{ - LogTagsMapping: v2.LogTagsMapping{ - Env: eg.LogTagsMapping.Env, - Service: eg.LogTagsMapping.Service, - Release: eg.LogTagsMapping.Release, - }, - QueryFilter: eg.QueryFilter, - }, - } -} - -func (m *v1tov2) migrateAsyncSearch(a v1.AsyncSearch) *v2.AsyncSearch { - if len(a.AdminUsers) == 0 && a.ListQueryLengthLimit == 0 { - return nil - } - - return &v2.AsyncSearch{ - SeqDBID: v2.DefaultSeqDBClientID, - Options: v2.AsyncSearchOptions{ - AdminUsers: a.AdminUsers, - ListQueryLengthLimit: a.ListQueryLengthLimit, - }, - } -} - -func (m *v1tov2) migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { - dst := &v2.MassExport{ - SeqDBID: v2.DefaultSeqDBClientID, - BatchSize: me.BatchSize, - WorkersCount: me.WorkersCount, - TasksChannelSize: me.TasksChannelSize, - PartLength: me.PartLength, - URLPrefix: me.URLPrefix, - AllowedUsers: me.AllowedUsers, - FileStore: m.migrateFileStore(me.FileStore), - DownloadParams: m.migrateDownloadParams(me.SeqProxyDownloader), - } - - if me.SessionStore != nil { - cfg.Cache.Redis = append(cfg.Cache.Redis, m.migrateRedis(&me.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) - - dst.SessionStore = &v2.SessionStore{ - RedisID: v2.DefaultMassExportRedisID, - ExportLifetime: me.SessionStore.ExportLifetime, - } - } - - return dst -} - -func (m *v1tov2) migrateFileStore(fs *v1.FileStore) *v2.FileStore { - if fs == nil { - return nil - } - - dst := &v2.FileStore{} - if fs.S3 != nil { - dst.S3 = &v2.S3{ - Endpoint: fs.S3.Endpoint, - AccessKeyID: fs.S3.AccessKeyID, - SecretAccessKey: fs.S3.SecretAccessKey, - BucketName: fs.S3.BucketName, - EnableSSl: fs.S3.EnableSSl, - } - } - - return dst -} - -func (m *v1tov2) migrateDownloadParams(src *v1.SeqProxyDownloader) *v2.DownloadParams { - if src == nil { - return nil - } - - return &v2.DownloadParams{ - Delay: src.Delay, - InitialRetryBackoff: src.InitialRetryBackoff, - MaxRetryBackoff: src.MaxRetryBackoff, - } -} - -func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { - if src == nil { - return nil - } - - _, redisID := m.defaultCacheIDs(v1Cache) - return &v2.Admin{ - Options: v2.AdminOptions{ - Cache: &v2.HandlerCache{ - ID: redisID, - TTL: src.CacheTTL, - }, - SuperUsers: src.SuperUsers, - }, - } -} - -func (m *v1tov2) defaultCacheIDs(src v1.Cache) (cacheID, redisID string) { - hasInmem := !m.isInmemEmpty(src.Inmemory) - hasRedis := src.Redis != nil - - switch { - case hasInmem && hasRedis: +// func (m *v1tov2) migrateClients(src v1.Config) v2.Clients { +// dst := v2.Clients{} +// +// if src.Clients != nil { +// if len(src.Clients.SeqDB) > 0 { +// dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) +// for i := range src.Clients.SeqDB { +// dst.SeqDB = append(dst.SeqDB, m.migrateSeqDBClient(&src.Clients.SeqDB[i])) +// } +// } else { +// dst.SeqDB = []v2.SeqDBClient{{ +// ID: v2.DefaultSeqDBClientID, +// Timeout: src.Clients.SeqDBTimeout, +// AvgDocSize: src.Clients.SeqDBAvgDocSize, +// Addrs: src.Clients.SeqDBAddrs, +// RequestRetries: src.Clients.RequestRetries, +// InitialRetryBackoff: src.Clients.InitialRetryBackoff, +// MaxRetryBackoff: src.Clients.MaxRetryBackoff, +// ClientMode: src.Clients.ProxyClientMode, +// GRPCParams: m.migrateGRPCParams(src.Clients.GRPCKeepaliveParams), +// }} +// } +// } +// +// if src.Server != nil && src.Server.CH != nil { +// ch := src.Server.CH +// dst.ClickHouse = []v2.CHClient{{ +// ID: v2.DefaultCHClientID, +// Addrs: ch.Addrs, +// Database: ch.Database, +// Username: ch.Username, +// Password: ch.Password, +// Sharded: ch.Sharded, +// DialTimeout: ch.DialTimeout, +// ReadTimeout: ch.ReadTimeout, +// }} +// } +// +// return dst +// } +// +// func (m *v1tov2) migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { +// return v2.SeqDBClient{ +// ID: src.ID, +// Timeout: src.Timeout, +// AvgDocSize: src.AvgDocSize, +// Addrs: src.Addrs, +// RequestRetries: src.RequestRetries, +// InitialRetryBackoff: src.InitialRetryBackoff, +// MaxRetryBackoff: src.MaxRetryBackoff, +// ClientMode: src.ClientMode, +// GRPCParams: m.migrateGRPCParams(src.GRPCKeepaliveParams), +// } +// } +// +// func (m *v1tov2) migrateDB(src *v1.Server) *v2.DB { +// if src == nil || src.DB == nil { +// return nil +// } +// +// return &v2.DB{ +// Name: src.DB.Name, +// Host: src.DB.Host, +// Port: src.DB.Port, +// Pass: src.DB.Pass, +// User: src.DB.User, +// RequestTimeout: src.DB.RequestTimeout, +// ConnectionPoolCapacity: src.DB.ConnectionPoolCapacity, +// UsePreparedStatements: src.DB.UsePreparedStatements, +// } +// } +// +// func (m *v1tov2) migrateCache(src v1.Cache) v2.Cache { +// // in config v1 inmemory cache is always created +// cache := v2.Cache{ +// Inmemory: []v2.InmemoryCache{{ +// ID: v2.DefaultInmemCacheID, +// NumCounters: src.Inmemory.NumCounters, +// MaxCost: src.Inmemory.MaxCost, +// BufferItems: src.Inmemory.BufferItems, +// }}, +// } +// +// if src.Redis != nil { +// cache.Redis = append(cache.Redis, +// m.migrateRedis(src.Redis, v2.DefaultRedisID, v2.DefaultInmemCacheID), +// m.migrateRedis(src.Redis, v2.DefaultRedis2ID, ""), +// ) +// } +// +// return cache +// } +// +// func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { +// return v2.Redis{ +// ID: id, +// WithInmemID: withInmemID, +// Addr: src.Addr, +// Username: src.Username, +// Password: src.Password, +// Timeout: src.Timeout, +// MaxRetries: src.MaxRetries, +// MinRetryBackoff: src.MinRetryBackoff, +// MaxRetryBackoff: src.MaxRetryBackoff, +// KeyPrefix: src.KeyPrefix, +// } +// } +// +// func (m *v1tov2) migrateHandlers(src *v1.Handlers, cfg *v2.Config, v1Cache v1.Cache) v2.Handlers { +// if src == nil { +// return v2.Handlers{} +// } +// +// dst := v2.Handlers{ +// SeqAPI: m.migrateSeqAPI(src.SeqAPI, v1Cache), +// ErrorGroups: m.migrateErrorGroups(src.ErrorGroups), +// AsyncSearch: m.migrateAsyncSearch(src.AsyncSearch), +// Admin: m.migrateAdmin(src.Admin, v1Cache), +// } +// +// if src.MassExport != nil { +// dst.MassExport = m.migrateMassExport(src.MassExport, cfg) +// } +// +// return dst +// } +// +// func (m *v1tov2) migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { +// cacheID, redisID := m.defaultCacheIDs(v1Cache) +// dst := v2.SeqAPI{ +// Envs: m.migrateSeqAPIEnvs(src.Envs, cacheID, redisID), +// DefaultEnv: src.DefaultEnv, +// Options: v2.SeqAPIOptions{ +// Caches: v2.SeqAPICaches{ +// Events: v2.HandlerCache{ +// ID: cacheID, +// TTL: src.EventsCacheTTL, +// }, +// LogsLifespan: v2.HandlerCache{ +// ID: redisID, +// TTL: src.LogsLifespanCacheTTL, +// }, +// Fields: v2.HandlerCache{ +// ID: cacheID, +// TTL: src.FieldsCacheTTL, +// }, +// }, +// }, +// } +// +// if len(src.Envs) == 0 { +// dst.SeqDBID = v2.DefaultSeqDBClientID +// } +// +// if src.SeqAPIOptions != nil { +// dst.Options = m.migrateSeqAPIOptions(src.SeqAPIOptions, cacheID, redisID) +// } +// +// return dst +// } +// +// func (m *v1tov2) migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv, cacheID, redisID string) map[string]v2.SeqAPIEnv { +// if len(envs) == 0 { +// return nil +// } +// +// dst := make(map[string]v2.SeqAPIEnv, len(envs)) +// for name, cfg := range envs { +// env := v2.SeqAPIEnv{ +// SeqDBID: cfg.SeqDB, +// } +// if cfg.Options != nil { +// envOptions := m.migrateSeqAPIOptions(cfg.Options, cacheID, redisID) +// env.Options = &envOptions +// } +// +// dst[name] = env +// } +// +// return dst +// } +// +// func (m *v1tov2) migrateSeqAPIOptions(options *v1.SeqAPIOptions, cacheID, redisID string) v2.SeqAPIOptions { +// return v2.SeqAPIOptions{ +// Limits: v2.SeqAPILimits{ +// MaxSearchLimit: options.MaxSearchLimit, +// MaxSearchTotal: options.MaxSearchTotalLimit, +// MaxSearchOffset: options.MaxSearchOffsetLimit, +// MaxExportLimit: options.MaxExportLimit, +// SeqCLIMaxSearchLimit: options.SeqCLIMaxSearchLimit, +// MaxParallelExportRequests: options.MaxParallelExportRequests, +// MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, +// MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, +// }, +// Masking: m.migrateMasking(options.Masking), +// PinnedFields: m.migrateFields(options.PinnedFields), +// SystemFields: m.migrateFields(options.SystemFields), +// Caches: v2.SeqAPICaches{ +// Events: v2.HandlerCache{ +// ID: cacheID, +// TTL: options.EventsCacheTTL, +// }, +// LogsLifespan: v2.HandlerCache{ +// ID: redisID, +// TTL: options.LogsLifespanCacheTTL, +// }, +// Fields: v2.HandlerCache{ +// ID: cacheID, +// TTL: options.FieldsCacheTTL, +// }, +// }, +// } +// } +// +// func (m *v1tov2) migrateFields(fs []v1.Field) []v2.Field { +// if fs == nil { +// return nil +// } +// +// dst := make([]v2.Field, len(fs)) +// for i, f := range fs { +// dst[i] = v2.Field{ +// Name: f.Name, +// Type: f.Type, +// } +// } +// +// return dst +// } +// +// func (m *v1tov2) migrateMasking(src *v1.Masking) *v2.Masking { +// if src == nil { +// return nil +// } +// +// return &v2.Masking{ +// Masks: m.migrateMasks(src.Masks), +// ProcessFields: src.ProcessFields, +// IgnoreFields: src.IgnoreFields, +// } +// } +// +// func (m *v1tov2) migrateMasks(ms []v1.Mask) []v2.Mask { +// if ms == nil { +// return nil +// } +// +// dst := make([]v2.Mask, len(ms)) +// for i := range ms { +// mask := &ms[i] +// dst[i] = v2.Mask{ +// Re: mask.Re, +// Groups: mask.Groups, +// Mode: mask.Mode, +// ReplaceWord: mask.ReplaceWord, +// ProcessFields: mask.ProcessFields, +// IgnoreFields: mask.IgnoreFields, +// FieldFilters: m.migrateFieldFilters(mask.FieldFilters), +// } +// } +// +// return dst +// } +// +// func (m *v1tov2) migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { +// if src == nil { +// return nil +// } +// +// dst := &v2.FieldFilterSet{ +// Condition: src.Condition, +// } +// if src.Filters != nil { +// dst.Filters = make([]v2.FieldFilter, len(src.Filters)) +// for i, f := range src.Filters { +// dst.Filters[i] = v2.FieldFilter{ +// Field: f.Field, +// Mode: f.Mode, +// Values: f.Values, +// } +// } +// } +// +// return dst +// } +// +// func (m *v1tov2) migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { +// if m.isEgEmpty(eg) { +// return nil +// } +// +// return &v2.ErrorGroups{ +// CHID: v2.DefaultCHClientID, +// Options: v2.ErrorGroupsOptions{ +// LogTagsMapping: v2.LogTagsMapping{ +// Env: eg.LogTagsMapping.Env, +// Service: eg.LogTagsMapping.Service, +// Release: eg.LogTagsMapping.Release, +// }, +// QueryFilter: eg.QueryFilter, +// }, +// } +// } +// +// func (m *v1tov2) migrateAsyncSearch(a v1.AsyncSearch) *v2.AsyncSearch { +// if len(a.AdminUsers) == 0 && a.ListQueryLengthLimit == 0 { +// return nil +// } +// +// return &v2.AsyncSearch{ +// SeqDBID: v2.DefaultSeqDBClientID, +// Options: v2.AsyncSearchOptions{ +// AdminUsers: a.AdminUsers, +// ListQueryLengthLimit: a.ListQueryLengthLimit, +// }, +// } +// } +// +// func (m *v1tov2) migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { +// dst := &v2.MassExport{ +// SeqDBID: v2.DefaultSeqDBClientID, +// Options: v2.MassExportOptions{ +// BatchSize: me.BatchSize, +// WorkersCount: me.WorkersCount, +// TasksChannelSize: me.TasksChannelSize, +// PartLength: me.PartLength, +// URLPrefix: me.URLPrefix, +// AllowedUsers: me.AllowedUsers, +// FileStore: m.migrateFileStore(me.FileStore), +// DownloadParams: m.migrateDownloadParams(me.SeqProxyDownloader), +// }, +// } +// +// if me.SessionStore != nil { +// cfg.Cache.Redis = append(cfg.Cache.Redis, m.migrateRedis(&me.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) +// +// dst.Options.SessionStore = v2.SessionStore{ +// RedisID: v2.DefaultMassExportRedisID, +// ExportLifetime: me.SessionStore.ExportLifetime, +// } +// } +// +// return dst +// } +// +// func (m *v1tov2) migrateFileStore(fs *v1.FileStore) v2.FileStore { +// if fs == nil || fs.S3 == nil { +// return v2.FileStore{} +// } +// +// return v2.FileStore{ +// S3: &v2.S3{ +// Endpoint: fs.S3.Endpoint, +// AccessKeyID: fs.S3.AccessKeyID, +// SecretAccessKey: fs.S3.SecretAccessKey, +// BucketName: fs.S3.BucketName, +// EnableSSL: fs.S3.EnableSSl, +// }, +// } +// } +// +// func (m *v1tov2) migrateDownloadParams(src *v1.SeqProxyDownloader) v2.DownloadParams { +// if src == nil { +// return v2.DownloadParams{} +// } +// +// return v2.DownloadParams{ +// Delay: src.Delay, +// InitialRetryBackoff: src.InitialRetryBackoff, +// MaxRetryBackoff: src.MaxRetryBackoff, +// } +// } +// +// func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { +// if src == nil { +// return nil +// } +// +// dst := &v2.Admin{ +// Options: v2.AdminOptions{ +// SuperUsers: src.SuperUsers, +// }, +// } +// +// _, redisID := m.defaultCacheIDs(v1Cache) +// if redisID != "" { +// dst.Options.Cache = &v2.HandlerCache{ +// ID: redisID, +// TTL: src.CacheTTL, +// } +// } +// +// return dst +// } +// +// defaultCacheIDs returns cache ids same as in v1. +// inmemWithRedisID - inmemory+redis or inmemory (v1 events, oidc, fields) +// redisOrInmemID - redis or inmemory (v1 logslifespan) +func (m *v1tov2) defaultCacheIDs(src v1.Cache) (inmemWithRedisID, redisOrInmemID string) { + if src.Redis != nil { return v2.DefaultRedisID, v2.DefaultRedis2ID - case hasRedis: - return v2.DefaultRedisID, v2.DefaultRedisID - case hasInmem: - return v2.DefaultInmemCacheID, "" - default: - return "", "" } + return v2.DefaultInmemCacheID, v2.DefaultInmemCacheID } -func (m *v1tov2) isInmemEmpty(src v1.InmemoryCache) bool { - return src.BufferItems == 0 && src.MaxCost == 0 && src.NumCounters == 0 -} - -func (m *v1tov2) isEgEmpty(src v1.ErrorGroups) bool { - return len(src.LogTagsMapping.Env) == 0 && len(src.LogTagsMapping.Service) == 0 && - len(src.LogTagsMapping.Release) == 0 && len(src.QueryFilter) == 0 -} +// +// func (m *v1tov2) isEgEmpty(src v1.ErrorGroups) bool { +// return len(src.LogTagsMapping.Env) == 0 && len(src.LogTagsMapping.Service) == 0 && +// len(src.LogTagsMapping.Release) == 0 && len(src.QueryFilter) == 0 +// } diff --git a/internal/app/config/migrate/v1_v2_test.go b/internal/app/config/migrate/v1_v2_test.go index 278a35f..83f1c00 100644 --- a/internal/app/config/migrate/v1_v2_test.go +++ b/internal/app/config/migrate/v1_v2_test.go @@ -1 +1,3 @@ package migrate + +// TODO diff --git a/internal/app/config/v1/config.go b/internal/app/config/v1/config.go index ac75aa2..929d81e 100644 --- a/internal/app/config/v1/config.go +++ b/internal/app/config/v1/config.go @@ -215,7 +215,7 @@ import ( // admin_users: // list_query_length_limit: // admin: -// admin_users: +// super_users: // cache_ttl: type Config struct { diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 8da85fc..58690fa 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -27,9 +27,12 @@ import ( // connection_timeout: // debug: // addr: +// read_timeout: +// read_header_timeout: +// write_timeout: // auth: -// options: -// oidc: +// options: +// oidc: // cache_id: // cache_secret_key: // skip_verify: @@ -42,10 +45,10 @@ import ( // allowed_clients: // jwt: // secret_key: -// envs: -// : -// options: # same as root options -// rate_limiters: +// envs: +// : +// options: # same as root options +// rate_limiting: // options: // : // default: @@ -69,18 +72,15 @@ import ( // avg_doc_size: // addrs: // request_retries: -// initial_retry_backoff: -// max_retry_backoff: +// backoff: +// initial_interval: +// max_interval // client_mode: // grpc_params: -// keep_alive: +// keepalive: // time: // timeout: // permit_without_stream: -// download_params: -// delay: -// initial_retry_backoff: -// max_retry_backoff: // clickhouse: // - id: // addrs: @@ -92,7 +92,6 @@ import ( // read_timeout: // handlers: // seq_api: -// seq_db_id: // options: // limits: // max_search_limit: @@ -119,29 +118,28 @@ import ( // values: // process_fields: // ignore_fields: -// caches: -// events: -// id: -// ttl: -// logs_lifespan: -// id: -// ttl: -// fields: -// id: -// ttl: -// pinned_fields: -// - name: -// type: -// system_fields: -// - name: -// type: +// caches: +// events: +// id: +// ttl: +// logs_lifespan: +// id: +// ttl: +// fields: +// id: +// ttl: +// pinned_fields: +// - name: +// type: +// system_fields: +// - name: +// type: // envs: // : // seq_db_id: // options: # same as root options // default_env: // error_groups: -// ch_id: // options: // log_tags_mapping: // env: @@ -155,43 +153,48 @@ import ( // options: # same as root options // default_env: // mass_export: -// seq_db_id: -// batch_size: -// workers_count: -// tasks_channel_size: -// part_length: -// url_prefix: -// allowed_users: -// file_store: -// s3: -// endpoint: -// access_key_id: -// secret_access_key: -// bucket_name: -// enable_ssl: -// session_store: -// redis_id: -// export_lifetime: -// download_params: -// delay: -// initial_retry_backoff: -// max_retry_backoff: +// options: +// workers_count: +// tasks_channel_size: +// part_length: +// url_prefix: +// allowed_users: +// file_store: +// s3: +// endpoint: +// access_key_id: +// secret_access_key: +// bucket_name: +// enable_ssl: +// session_store: +// lifetime: +// cache_id: +// download_params: +// batch_size: +// delay: +// delay_backoff: +// initial_interval: +// max_interval: +// envs: +// : +// seq_db_id: +// options: # same as root options +// default_env: // async_search: -// seq_db_id: // options: -// admin_users: -// list_query_length_limit: +// admin_users: +// list_query_length_limit: // envs: // : // seq_db_id: // options: # same as root options // default_env: -// admin: -// options: -// super_users: -// cache: -// id: -// ttl: +// admin: +// options: +// super_users: +// cache: +// id: +// ttl: // envs: // : // options: # same as root options @@ -200,8 +203,8 @@ import ( // name: // host: // port: -// pass: // user: +// pass: // request_timeout: // connection_pool_capacity: // use_prepared_statements: @@ -218,8 +221,8 @@ import ( // addr: // username: // password: -// timeout: // max_retries: +// timeout: // min_retry_backoff: // max_retry_backoff: @@ -254,19 +257,19 @@ type OIDC struct { CacheSecretKey string `yaml:"cache_secret_key"` SkipVerify bool `yaml:"skip_verify"` AuthURLs []string `yaml:"auth_urls"` - TLS *TLS `yaml:"tls"` AllowedClients []string `yaml:"allowed_clients"` + TLS *TLS `yaml:"tls"` } type DB struct { Name string `yaml:"name"` Host string `yaml:"host"` Port int64 `yaml:"port"` - Pass string `yaml:"pass"` User string `yaml:"user"` + Pass string `yaml:"pass"` RequestTimeout time.Duration `yaml:"request_timeout"` ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` - UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` + UsePreparedStatements *bool `yaml:"use_prepared_statements"` } func (db *DB) ConnString() string { @@ -281,17 +284,15 @@ type ( PerHandler bool `yaml:"per_handler"` } - UserToRateLimiter map[string]RateLimiter - - ApiRateLimiters struct { - Default RateLimiter `yaml:"default"` - SpecialUsers UserToRateLimiter `yaml:"spec_users"` + RateLimiters struct { + Default RateLimiter `yaml:"default"` + SpecialUsers map[string]RateLimiter `yaml:"spec_users"` } - ApiToRateLimiters map[string]ApiRateLimiters + ApiToRateLimiters map[string]RateLimiters ) -type InmemoryCache struct { +type Inmemory struct { ID string `yaml:"id"` NumCounters int64 `yaml:"num_counters"` MaxCost int64 `yaml:"max_cost"` @@ -305,19 +306,19 @@ type Redis struct { Addr string `yaml:"addr"` Username string `yaml:"username"` Password string `yaml:"password"` - Timeout time.Duration `yaml:"timeout"` MaxRetries int `yaml:"max_retries"` + Timeout time.Duration `yaml:"timeout"` MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` } type Cache struct { - Inmemory []InmemoryCache `yaml:"inmemory"` - Redis []Redis `yaml:"redis"` + Inmemory []Inmemory `yaml:"inmemory"` + Redis []Redis `yaml:"redis"` } -func (c *Cache) InmemByID(id string) *InmemoryCache { - i := slices.IndexFunc(c.Inmemory, func(inm InmemoryCache) bool { +func (c *Cache) InmemByID(id string) *Inmemory { + i := slices.IndexFunc(c.Inmemory, func(inm Inmemory) bool { return inm.ID == id }) if i == -1 { @@ -343,18 +344,18 @@ type S3 struct { AccessKeyID string `yaml:"access_key_id"` SecretAccessKey string `yaml:"secret_access_key"` BucketName string `yaml:"bucket_name"` - EnableSSl bool `yaml:"enable_ssl"` + EnableSSL bool `yaml:"enable_ssl"` } type DownloadParams struct { - Delay time.Duration `yaml:"delay"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + BatchSize uint64 `yaml:"batch_size"` + Delay time.Duration `yaml:"delay"` + DelayBackoff Backoff `yaml:"delay_backoff"` } type SessionStore struct { - RedisID string `yaml:"redis_id"` - ExportLifetime time.Duration `yaml:"export_lifetime"` + Lifetime time.Duration `yaml:"lifetime"` + CacheID string `yaml:"cache_id"` } type FileStore struct { @@ -362,16 +363,25 @@ type FileStore struct { } type MassExport struct { - SeqDBID string `yaml:"seq_db_id"` - BatchSize uint64 `yaml:"batch_size"` - WorkersCount int `yaml:"workers_count"` - TasksChannelSize int `yaml:"tasks_channel_size"` - PartLength time.Duration `yaml:"part_length"` - URLPrefix string `yaml:"url_prefix"` - AllowedUsers []string `yaml:"allowed_users"` - FileStore *FileStore `yaml:"file_store"` - SessionStore *SessionStore `yaml:"session_store"` - DownloadParams *DownloadParams `yaml:"download_params"` + Options MassExportOptions `yaml:"options"` + Envs map[string]MassExportEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type MassExportEnv struct { + SeqDBID string `yaml:"seq_db_id"` + Options *MassExportOptions `yaml:"options"` +} + +type MassExportOptions struct { + WorkersCount int `yaml:"workers_count"` + TasksChannelSize int `yaml:"tasks_channel_size"` + PartLength time.Duration `yaml:"part_length"` + URLPrefix string `yaml:"url_prefix"` + AllowedUsers []string `yaml:"allowed_users"` + FileStore FileStore `yaml:"file_store"` + SessionStore SessionStore `yaml:"session_store"` + DownloadParams DownloadParams `yaml:"download_params"` } type HTTP struct { @@ -388,7 +398,10 @@ type GRPC struct { } type Debug struct { - Addr string `yaml:"addr"` + Addr string `yaml:"addr"` + ReadHeaderTimeout time.Duration `yaml:"read_header_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` + WriteTimeout time.Duration `yaml:"write_timeout"` } type JWT struct { @@ -396,12 +409,8 @@ type JWT struct { } type Auth struct { - Options AuthOptions `yaml:"options"` - Envs map[string]AuthEnv `yaml:"envs"` -} - -type AuthEnv struct { - Options *AuthOptions `yaml:"options"` + Options AuthOptions `yaml:"options"` + Envs map[string]AuthOptions `yaml:"envs"` } type AuthOptions struct { @@ -410,20 +419,16 @@ type AuthOptions struct { } type Server struct { - HTTP HTTP `yaml:"http"` - GRPC GRPC `yaml:"grpc"` + HTTP *HTTP `yaml:"http"` + GRPC *GRPC `yaml:"grpc"` Debug Debug `yaml:"debug"` Auth *Auth `yaml:"auth"` - RateLimiters *RateLimiters `yaml:"rate_limiters"` -} - -type RateLimiters struct { - Options ApiToRateLimiters `yaml:"options"` - Envs map[string]RateLimitersEnv `yaml:"envs"` + RateLimiting *RateLimiting `yaml:"rate_limiting"` } -type RateLimitersEnv struct { - Options ApiToRateLimiters `yaml:"options"` +type RateLimiting struct { + Options ApiToRateLimiters `yaml:"options"` + Envs map[string]ApiToRateLimiters `yaml:"envs"` } type KeepaliveParams struct { @@ -442,20 +447,23 @@ type KeepaliveParams struct { } type GRPCParams struct { - Keepalive *KeepaliveParams `yaml:"keep_alive"` + Keepalive *KeepaliveParams `yaml:"keepalive"` } type SeqDBClient struct { - ID string `yaml:"id"` - Timeout time.Duration `yaml:"timeout"` - AvgDocSize int `yaml:"avg_doc_size"` - Addrs []string `yaml:"addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ClientMode string `yaml:"client_mode"` - GRPCParams *GRPCParams `yaml:"grpc_params"` - DownloadParams *DownloadParams `yaml:"download_params"` + ID string `yaml:"id"` + Timeout time.Duration `yaml:"timeout"` + AvgDocSize int `yaml:"avg_doc_size"` + Addrs []string `yaml:"addrs"` + RequestRetries int `yaml:"request_retries"` + RetriesBackoff *Backoff `yaml:"backoff"` + ClientMode string `yaml:"client_mode"` + GRPCParams *GRPCParams `yaml:"grpc_params"` +} + +type Backoff struct { + InitialInterval time.Duration `yaml:"initial_interval"` + MaxInterval time.Duration `yaml:"max_interval"` } type CHClient struct { @@ -486,7 +494,7 @@ func (c *Clients) ClickHouseByID(id string) *CHClient { } type Handlers struct { - SeqAPI SeqAPI `yaml:"seq_api"` + SeqAPI *SeqAPI `yaml:"seq_api"` MassExport *MassExport `yaml:"mass_export"` AsyncSearch *AsyncSearch `yaml:"async_search"` ErrorGroups *ErrorGroups `yaml:"error_groups"` @@ -514,7 +522,7 @@ type Field struct { } type SeqAPI struct { - SeqDBID string `yaml:"seq_db_id"` + // добавить проверку на len(SEQDB) == 1 и берем [0] Options SeqAPIOptions `yaml:"options"` Envs map[string]SeqAPIEnv `yaml:"envs"` DefaultEnv string `yaml:"default_env"` @@ -534,14 +542,14 @@ type SeqAPIOptions struct { } type SeqAPILimits struct { - MaxSearchLimit int32 `yaml:"max_search_limit,omitempty"` - MaxSearchTotal int64 `yaml:"max_search_total,omitempty"` - MaxSearchOffset int32 `yaml:"max_search_offset,omitempty"` - MaxExportLimit int32 `yaml:"max_export_limit,omitempty"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit,omitempty"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests,omitempty"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request,omitempty"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts,omitempty"` + MaxSearchLimit int `yaml:"max_search_limit"` + MaxSearchTotal int `yaml:"max_search_total"` + MaxSearchOffset int `yaml:"max_search_offset"` + MaxExportLimit int `yaml:"max_export_limit"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` } type SeqAPICaches struct { @@ -591,7 +599,6 @@ type LogTagsMapping struct { } type ErrorGroups struct { - CHID string `yaml:"ch_id"` Options ErrorGroupsOptions `yaml:"options"` Envs map[string]ErrorGroupsEnv `yaml:"envs"` DefaultEnv string `yaml:"default_env"` @@ -608,7 +615,6 @@ type ErrorGroupsOptions struct { } type AsyncSearch struct { - SeqDBID string `yaml:"seq_db_id"` Options AsyncSearchOptions `yaml:"options"` Envs map[string]AsyncSearchEnv `yaml:"envs"` DefaultEnv string `yaml:"default_env"` diff --git a/internal/app/config/v2/validate.go b/internal/app/config/v2/validate.go index 8ad63e1..77a1dcb 100644 --- a/internal/app/config/v2/validate.go +++ b/internal/app/config/v2/validate.go @@ -6,12 +6,13 @@ import ( ) const ( - DefaultSeqDBClientID = "default_seq_db" - DefaultCHClientID = "default_ch" - DefaultInmemCacheID = "default_inmem_cache" - DefaultRedisID = "default_redis" - DefaultRedis2ID = "default_redis_2" - DefaultMassExportRedisID = "mass_export" + MaxHTTPHeaderBytes = 1 << 12 // 4 KiB + + DefaultSeqDBClientID = "default_seq_db" + DefaultCHClientID = "default_ch" + DefaultInmemCacheID = "default_inmem_cache" + DefaultRedisID = "default_redis" + DefaultRedis2ID = "default_redis_2" SeqDBClientModeGRPC = "grpc" @@ -33,10 +34,13 @@ const ( minGRPCKeepaliveTimeout = 1 * time.Second defaultAsyncSearchListQueryLengthLimit = 1000 + defaultCORSAllowedOrigins = "*" defaultMaxSearchTotal = 1000000 defaultMaxSearchOffset = 1000000 defaultMaxExportLimit = 100000 + defaultMaxSearchLimit = 1000 + defaultSeqCLIMaxSearchLimit = 1000 defaultMaxAggregationsPerRequest = 1 defaultMaxBucketsPerAggregationTs = 200 defaultMaxParallelExportRequests = 1 @@ -57,298 +61,389 @@ const ( defaultUsePreparedStatements = true ) -func (cfg *Config) Validate() error { - if cfg.Server.GRPC.ConnectionTimeout <= 0 { - cfg.Server.GRPC.ConnectionTimeout = defaultGRPCConnectionTimeout - } - - if len(cfg.Clients.SeqDB) == 0 { - return fmt.Errorf("clients.seq_db must contain at least one client") - } +var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE"} - seqDBIDs := make(map[string]struct{}, len(cfg.Clients.SeqDB)) - for i := range cfg.Clients.SeqDB { - c := &cfg.Clients.SeqDB[i] - if c.ID == "" { - return fmt.Errorf("clients.seq_db[%d].id cannot be empty", i) - } - if _, ok := seqDBIDs[c.ID]; ok { - return fmt.Errorf("duplicate clients.seq_db.id %q", c.ID) - } - - seqDBIDs[c.ID] = struct{}{} - - if len(c.Addrs) == 0 { - return fmt.Errorf("clients.seq_db[%q].addrs cannot be empty", c.ID) - } - if c.Timeout <= 0 { - return fmt.Errorf("clients.seq_db[%q].timeout must be > 0", c.ID) - } - if c.ClientMode == "" { - c.ClientMode = SeqDBClientModeGRPC - } else if c.ClientMode != SeqDBClientModeGRPC { - return fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, SeqDBClientModeGRPC) - } - - if c.GRPCParams != nil { - if c.GRPCParams.Keepalive != nil { - c.GRPCParams.Keepalive.Time = max(c.GRPCParams.Keepalive.Time, minGRPCKeepaliveTime) - c.GRPCParams.Keepalive.Timeout = max(c.GRPCParams.Keepalive.Timeout, minGRPCKeepaliveTimeout) - } - } - } - - chIDs := make(map[string]struct{}, len(cfg.Clients.ClickHouse)) - for i := range cfg.Clients.ClickHouse { - ch := &cfg.Clients.ClickHouse[i] - if ch.ID == "" { - return fmt.Errorf("clients.clickhouse[%d].id cannot be empty", i) - } - if _, ok := chIDs[ch.ID]; ok { - return fmt.Errorf("duplicate clients.clickhouse.id %q", ch.ID) - } - - chIDs[ch.ID] = struct{}{} - - if ch.DialTimeout <= 0 { - ch.DialTimeout = defaultClickHouseDialTimeout - } - if ch.ReadTimeout <= 0 { - ch.ReadTimeout = defaultClickHouseReadTimeout - } - } - - inmemIDs := make(map[string]struct{}, len(cfg.Cache.Inmemory)) - for i := range cfg.Cache.Inmemory { - inm := &cfg.Cache.Inmemory[i] - if inm.ID == "" { - return fmt.Errorf("cache.inmemory[%d].id cannot be empty", i) - } - if _, ok := inmemIDs[inm.ID]; ok { - return fmt.Errorf("duplicate cache.inmemory.id %q", inm.ID) - } - - inmemIDs[inm.ID] = struct{}{} - - if inm.NumCounters <= 0 { - inm.NumCounters = defaultInmemCacheNumCounters - } - if inm.MaxCost <= 0 { - inm.MaxCost = defaultInmemCacheMaxCost - } - if inm.BufferItems <= 0 { - inm.BufferItems = defaultInmemCacheBufferItems - } - } - - redisIDs := make(map[string]struct{}, len(cfg.Cache.Redis)) - for i := range cfg.Cache.Redis { - r := &cfg.Cache.Redis[i] - if r.ID == "" { - return fmt.Errorf("cache.redis[%d].id cannot be empty", i) - } - if _, ok := redisIDs[r.ID]; ok { - return fmt.Errorf("duplicate cache.redis.id %q", r.ID) - } - - redisIDs[r.ID] = struct{}{} - - if r.WithInmemID != "" { - if _, ok := inmemIDs[r.WithInmemID]; !ok { - return fmt.Errorf("cache.redis[%q].with_inmem_id %q not found in cache.inmemory", r.ID, r.WithInmemID) - } - } - } - - if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { - v := defaultUsePreparedStatements - cfg.DB.UsePreparedStatements = &v - } - - if cfg.Server.Auth != nil && cfg.Server.Auth.Options.OIDC != nil { - oidc := cfg.Server.Auth.Options.OIDC - hasCacheKey := oidc.CacheSecretKey != "" - hasCacheID := oidc.CacheID != "" - - switch { - case hasCacheKey && !hasCacheID: - return fmt.Errorf("auth.oidc.cache_secret_key is set but auth.oidc.cache_id is empty") - case !hasCacheKey && hasCacheID: - return fmt.Errorf("auth.oidc.cache_id is set but auth.oidc.cache_secret_key is empty") - case hasCacheID && hasCacheKey: - if cfg.Cache.InmemByID(oidc.CacheID) == nil && cfg.Cache.RedisByID(oidc.CacheID) == nil { - return fmt.Errorf("auth.oidc.cache_id %q not found", oidc.CacheID) - } - } - } - - setSeqAPIOptionsDefaults(&cfg.Handlers.SeqAPI.Options) - - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - if cfg.Handlers.SeqAPI.SeqDBID != "" { - return fmt.Errorf("handlers.seq_api.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") - } - if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return fmt.Errorf("handlers.seq_api.default_env must be specified when envs is used") - } - if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return fmt.Errorf("handlers.seq_api.default_env %q not found in envs", cfg.Handlers.SeqAPI.DefaultEnv) - } - - for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if envConfig.SeqDBID == "" { - return fmt.Errorf("handlers.seq_api.envs[%q].seq_db_id cannot be empty", envName) - } - if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) - } - setSeqAPIOptionsDefaults(envConfig.Options) - } - } else { - if cfg.Handlers.SeqAPI.SeqDBID == "" { - return fmt.Errorf("handlers.seq_api.seq_db_id cannot be empty when envs is not used") - } - if _, ok := seqDBIDs[cfg.Handlers.SeqAPI.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.seq_api.seq_db_id %q", cfg.Handlers.SeqAPI.SeqDBID) - } +func (cfg *Config) Validate() error { + // seqDBIDs, chIDs, err := validateClients(&cfg.Clients) + // if err != nil { + // return err + // } + // cacheIDs, err := validateCache(&cfg.Cache) + // if err != nil { + // return err + // } + + validateDB(cfg.DB) + + if err := validateServer(&cfg.Server); err != nil { + return err } + // if err := validateSeqAPI(cfg.Handlers.SeqAPI, seqDBIDs, cacheIDs); err != nil { + // return err + // } + // if err := validateAdmin(cfg.Handlers.Admin, &cfg.Cache); err != nil { + // return err + // } + + // if cfg.Handlers.MassExport != nil { + // me := cfg.Handlers.MassExport + // + // if len(me.Envs) > 0 { + // if me.SeqDBID != "" { + // return fmt.Errorf("handlers.mass_export.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + // } + // if me.DefaultEnv == "" { + // return fmt.Errorf("handlers.mass_export.default_env must be specified when envs is used") + // } + // if _, exists := me.Envs[me.DefaultEnv]; !exists { + // return fmt.Errorf("handlers.mass_export.default_env %q not found in envs", me.DefaultEnv) + // } + // + // for envName, envConfig := range me.Envs { + // if envConfig.SeqDBID == "" { + // return fmt.Errorf("handlers.mass_export.envs[%q].seq_db_id cannot be empty", envName) + // } + // if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + // return fmt.Errorf("unknown handlers.mass_export.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + // } + // + // if envConfig.Options != nil { + // if envConfig.Options.SessionStore.RedisID == "" { + // return fmt.Errorf("handlers.mass_export.envs[%q].options.session_store.redis_id cannot be empty", envName) + // } + // if _, ok := redisIDs[envConfig.Options.SessionStore.RedisID]; !ok { + // return fmt.Errorf("unknown handlers.mass_export.envs[%q].options.session_store.redis_id %q", envName, envConfig.Options.SessionStore.RedisID) + // } + // if envConfig.Options.FileStore.S3 == nil { + // return fmt.Errorf("handlers.mass_export.envs[%q].options.file_store.s3 cannot be empty", envName) + // } + // } + // } + // } else { + // if me.SeqDBID == "" { + // return fmt.Errorf("handlers.mass_export.seq_db_id cannot be empty when envs is not used") + // } + // if _, ok := seqDBIDs[me.SeqDBID]; !ok { + // return fmt.Errorf("unknown handlers.mass_export.seq_db_id %q", me.SeqDBID) + // } + // if me.Options.SessionStore.RedisID == "" { + // return fmt.Errorf("handlers.mass_export.options.session_store.redis_id cannot be empty") + // } + // if _, ok := redisIDs[me.Options.SessionStore.RedisID]; !ok { + // return fmt.Errorf("unknown handlers.mass_export.options.session_store.redis_id %q", me.Options.SessionStore.RedisID) + // } + // } + // } + // + // if cfg.Handlers.AsyncSearch != nil { + // as := cfg.Handlers.AsyncSearch + // + // if as.Options.ListQueryLengthLimit <= 0 { + // as.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + // } + // + // if len(as.Envs) > 0 { + // if as.SeqDBID != "" { + // return fmt.Errorf("handlers.async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + // } + // if as.DefaultEnv == "" { + // return fmt.Errorf("handlers.async_search.default_env must be specified when envs is used") + // } + // if _, ok := as.Envs[as.DefaultEnv]; !ok { + // return fmt.Errorf("handlers.async_search.default_env %q not found in envs", as.DefaultEnv) + // } + // + // for envName, envConfig := range as.Envs { + // if envConfig.SeqDBID == "" { + // return fmt.Errorf("handlers.async_search.envs[%q].seq_db_id cannot be empty", envName) + // } + // if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + // return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + // } + // if envConfig.Options.ListQueryLengthLimit <= 0 { + // envConfig.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + // } + // } + // } else { + // if as.SeqDBID == "" { + // return fmt.Errorf("handlers.async_search.seq_db_id cannot be empty when envs is not used") + // } + // if _, ok := seqDBIDs[as.SeqDBID]; !ok { + // return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", as.SeqDBID) + // } + // } + // } + // + // if cfg.Handlers.ErrorGroups != nil { + // eg := cfg.Handlers.ErrorGroups + // if len(eg.Envs) > 0 { + // if eg.CHID != "" { + // return fmt.Errorf("handlers.error_groups.ch_id must be empty when envs is used. Put ch_id inside each env") + // } + // if eg.DefaultEnv == "" { + // return fmt.Errorf("handlers.error_groups.default_env must be specified when envs is used") + // } + // if _, ok := eg.Envs[eg.DefaultEnv]; !ok { + // return fmt.Errorf("handlers.error_groups.default_env %q not found in envs", eg.DefaultEnv) + // } + // + // for envName, envConfig := range eg.Envs { + // if envConfig.CHID == "" { + // return fmt.Errorf("handlers.error_groups.envs[%q].ch_id cannot be empty", envName) + // } + // if _, ok := chIDs[envConfig.CHID]; !ok { + // return fmt.Errorf("unknown handlers.error_groups.envs[%q].ch_id %q", envName, envConfig.CHID) + // } + // } + // } else { + // if cfg.Handlers.ErrorGroups.CHID == "" { + // return fmt.Errorf("handlers.error_groups.ch_id cannot be empty when envs is not used") + // } + // if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { + // return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) + // } + // } + // } - if cfg.Handlers.MassExport != nil { - if cfg.Handlers.MassExport.SeqDBID == "" { - return fmt.Errorf("handlers.mass_export.seq_db_id cannot be empty") - } - if _, ok := seqDBIDs[cfg.Handlers.MassExport.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.mass_export.seq_db_id %q", cfg.Handlers.MassExport.SeqDBID) - } + return nil +} - if cfg.Handlers.MassExport.SessionStore != nil { - if cfg.Handlers.MassExport.SessionStore.RedisID == "" { - return fmt.Errorf("handlers.mass_export.session_store.redis_id cannot be empty") - } - if _, ok := redisIDs[cfg.Handlers.MassExport.SessionStore.RedisID]; !ok { - return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) - } - } +func validateServer(server *Server) error { + if server.HTTP == nil && server.GRPC == nil { + return fmt.Errorf("server: at least one of http or grpc must be set") } - if cfg.Handlers.AsyncSearch != nil { - as := cfg.Handlers.AsyncSearch - - if as.Options.ListQueryLengthLimit <= 0 { - as.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + if server.HTTP.CORS != nil { + if len(server.HTTP.CORS.AllowedOrigins) == 0 { + server.HTTP.CORS.AllowedOrigins = []string{defaultCORSAllowedOrigins} } - - if len(as.Envs) > 0 { - if as.SeqDBID != "" { - return fmt.Errorf("handlers.async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") - } - if as.DefaultEnv == "" { - return fmt.Errorf("handlers.async_search.default_env must be specified when envs is used") - } - if _, ok := as.Envs[as.DefaultEnv]; !ok { - return fmt.Errorf("handlers.async_search.default_env %q not found in envs", as.DefaultEnv) - } - - for envName, envConfig := range as.Envs { - if envConfig.SeqDBID == "" { - return fmt.Errorf("handlers.async_search.envs[%q].seq_db_id cannot be empty", envName) - } - if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) - } - if envConfig.Options.ListQueryLengthLimit <= 0 { - envConfig.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit - } - } - } else { - if as.SeqDBID == "" { - return fmt.Errorf("handlers.async_search.seq_db_id cannot be empty when envs is not used") - } - if _, ok := seqDBIDs[as.SeqDBID]; !ok { - return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", as.SeqDBID) - } - } - } - - if cfg.Handlers.ErrorGroups != nil { - eg := cfg.Handlers.ErrorGroups - if len(eg.Envs) > 0 { - if eg.CHID != "" { - return fmt.Errorf("handlers.error_groups.ch_id must be empty when envs is used. Put ch_id inside each env") - } - if eg.DefaultEnv == "" { - return fmt.Errorf("handlers.error_groups.default_env must be specified when envs is used") - } - if _, ok := eg.Envs[eg.DefaultEnv]; !ok { - return fmt.Errorf("handlers.error_groups.default_env %q not found in envs", eg.DefaultEnv) - } - - for envName, envConfig := range eg.Envs { - if envConfig.CHID == "" { - return fmt.Errorf("handlers.error_groups.envs[%q].ch_id cannot be empty", envName) - } - if _, ok := chIDs[envConfig.CHID]; !ok { - return fmt.Errorf("unknown handlers.error_groups.envs[%q].ch_id %q", envName, envConfig.CHID) - } - } - } else { - if cfg.Handlers.ErrorGroups.CHID == "" { - return fmt.Errorf("handlers.error_groups.ch_id cannot be empty when envs is not used") - } - if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { - return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) - } + if len(server.HTTP.CORS.AllowedMethods) == 0 { + server.HTTP.CORS.AllowedMethods = defaultCORSAllowedMethods } } - if cfg.Handlers.Admin != nil { - admin := cfg.Handlers.Admin - if admin.Options.Cache.TTL <= 0 { - admin.Options.Cache.TTL = defaultAdminCacheTTL - } - if admin.Options.Cache.ID != "" { - redisCfg := cfg.Cache.RedisByID(admin.Options.Cache.ID) - if redisCfg == nil { - return fmt.Errorf("unknown handlers.admin.redis_id %q", admin.Options.Cache.ID) - } - if redisCfg.WithInmemID != "" { - return fmt.Errorf("handlers.admin.redis_id %q: with_inmem_id is not allowed", admin.Options.Cache.ID) - } - } + if server.GRPC != nil && server.GRPC.ConnectionTimeout <= 0 { + server.GRPC.ConnectionTimeout = defaultGRPCConnectionTimeout } return nil } -func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { - if options.Limits.MaxAggregationsPerRequest <= 0 { - options.Limits.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest - } - if options.Limits.MaxBucketsPerAggregationTs <= 0 { - options.Limits.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs - } - if options.Limits.MaxParallelExportRequests <= 0 { - options.Limits.MaxParallelExportRequests = defaultMaxParallelExportRequests - } - if options.Limits.MaxSearchTotal <= 0 { - options.Limits.MaxSearchTotal = defaultMaxSearchTotal - } - if options.Limits.MaxSearchOffset <= 0 { - options.Limits.MaxSearchOffset = defaultMaxSearchOffset - } - if options.Limits.MaxExportLimit <= 0 { - options.Limits.MaxExportLimit = defaultMaxExportLimit - } - if options.Limits.MaxSearchLimit <= 0 { - options.Limits.MaxSearchLimit = 1000 // Надо дефолт задать нам, до этого его не было - } - if options.Limits.SeqCLIMaxSearchLimit <= 0 { - options.Limits.SeqCLIMaxSearchLimit = 1000 // Сюда тоже или в них не было небходимости вообще, чтобы они были по нулям - } - if options.Caches.Events.TTL <= 0 { - options.Caches.Events.TTL = defaultEventsCacheTTL - } - if options.Caches.LogsLifespan.TTL <= 0 { - options.Caches.LogsLifespan.TTL = defaultLogsLifespanCacheTTL +// func validateClients(clients *Clients) (seqDBIDs, chIDs map[string]struct{}, err error) { +// if len(clients.SeqDB) == 0 { +// return nil, nil, fmt.Errorf("clients.seq_db must contain at least one client") +// } +// +// seqDBIDs = make(map[string]struct{}, len(clients.SeqDB)) +// for i := range clients.SeqDB { +// c := &clients.SeqDB[i] +// if c.ID == "" { +// return nil, nil, fmt.Errorf("clients.seq_db[%d].id cannot be empty", i) +// } +// if _, ok := seqDBIDs[c.ID]; ok { +// return nil, nil, fmt.Errorf("duplicate clients.seq_db.id %q", c.ID) +// } +// +// seqDBIDs[c.ID] = struct{}{} +// +// if len(c.Addrs) == 0 { +// return nil, nil, fmt.Errorf("clients.seq_db[%q].addrs cannot be empty", c.ID) +// } +// if c.Timeout <= 0 { +// return nil, nil, fmt.Errorf("clients.seq_db[%q].timeout must be > 0", c.ID) +// } +// if c.ClientMode == "" { +// c.ClientMode = SeqDBClientModeGRPC +// } else if c.ClientMode != SeqDBClientModeGRPC { +// return nil, nil, fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, SeqDBClientModeGRPC) +// } +// +// if c.GRPCParams != nil { +// if c.GRPCParams.Keepalive != nil { +// c.GRPCParams.Keepalive.Time = max(c.GRPCParams.Keepalive.Time, minGRPCKeepaliveTime) +// c.GRPCParams.Keepalive.Timeout = max(c.GRPCParams.Keepalive.Timeout, minGRPCKeepaliveTimeout) +// } +// } +// } +// +// chIDs = make(map[string]struct{}, len(clients.ClickHouse)) +// for i := range clients.ClickHouse { +// ch := &clients.ClickHouse[i] +// if ch.ID == "" { +// return nil, nil, fmt.Errorf("clients.clickhouse[%d].id cannot be empty", i) +// } +// if _, ok := chIDs[ch.ID]; ok { +// return nil, nil, fmt.Errorf("duplicate clients.clickhouse.id %q", ch.ID) +// } +// +// chIDs[ch.ID] = struct{}{} +// +// if ch.DialTimeout <= 0 { +// ch.DialTimeout = defaultClickHouseDialTimeout +// } +// if ch.ReadTimeout <= 0 { +// ch.ReadTimeout = defaultClickHouseReadTimeout +// } +// } +// +// return seqDBIDs, chIDs, nil +// } +// +// func validateCache(cache *Cache) (map[string]struct{}, error) { +// inmemIDs := make(map[string]struct{}, len(cache.Inmemory)) +// for i := range cache.Inmemory { +// inm := &cache.Inmemory[i] +// if inm.ID == "" { +// return nil, fmt.Errorf("cache.inmemory[%d].id cannot be empty", i) +// } +// if _, ok := inmemIDs[inm.ID]; ok { +// return nil, fmt.Errorf("duplicate cache.inmemory.id %q", inm.ID) +// } +// +// inmemIDs[inm.ID] = struct{}{} +// +// if inm.NumCounters <= 0 { +// inm.NumCounters = defaultInmemCacheNumCounters +// } +// if inm.MaxCost <= 0 { +// inm.MaxCost = defaultInmemCacheMaxCost +// } +// if inm.BufferItems <= 0 { +// inm.BufferItems = defaultInmemCacheBufferItems +// } +// } +// +// cacheIDs := make(map[string]struct{}, len(cache.Inmemory)+len(cache.Redis)) +// for i := range cache.Redis { +// r := &cache.Redis[i] +// if r.ID == "" { +// return nil, fmt.Errorf("cache.redis[%d].id cannot be empty", i) +// } +// if _, ok := cacheIDs[r.ID]; ok { +// return nil, fmt.Errorf("duplicate cache.redis.id %q", r.ID) +// } +// +// cacheIDs[r.ID] = struct{}{} +// +// if r.WithInmemID != "" { +// if _, ok := inmemIDs[r.WithInmemID]; !ok { +// return nil, fmt.Errorf("cache.redis[%q].with_inmem_id %q not found in cache.inmemory", r.ID, r.WithInmemID) +// } +// } +// +// for id := range inmemIDs { +// if !slices.ContainsFunc(cache.Redis, func(r Redis) bool { +// return r.WithInmemID == id +// }) { +// cacheIDs[id] = struct{}{} +// } +// } +// } +// +// return cacheIDs, nil +// } + +func validateDB(db *DB) { + if db != nil && db.UsePreparedStatements == nil { + v := defaultUsePreparedStatements + db.UsePreparedStatements = &v } } + +// func validateSeqAPI(seqAPI *SeqAPI, seqDBIDs, cacheIDs map[string]struct{}) error { +// setSeqAPIOptionsDefaults(&seqAPI.Options) +// +// _ = cacheIDs // Проверки потом нафигачу на кэши, пусть весит пока голенький +// if len(seqAPI.Envs) > 0 { +// if seqAPI.SeqDBID != "" { +// return fmt.Errorf("handlers.seq_api.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") +// } +// if seqAPI.DefaultEnv == "" { +// return fmt.Errorf("handlers.seq_api.default_env must be specified when envs is used") +// } +// if _, exists := seqAPI.Envs[seqAPI.DefaultEnv]; !exists { +// return fmt.Errorf("handlers.seq_api.default_env %q not found in envs", seqAPI.DefaultEnv) +// } +// +// for envName, envConfig := range seqAPI.Envs { +// if envConfig.SeqDBID == "" { +// return fmt.Errorf("handlers.seq_api.envs[%q].seq_db_id cannot be empty", envName) +// } +// if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { +// return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) +// } +// setSeqAPIOptionsDefaults(envConfig.Options) +// } +// } else { +// if seqAPI.SeqDBID == "" { +// return fmt.Errorf("handlers.seq_api.seq_db_id cannot be empty when envs is not used") +// } +// if _, ok := seqDBIDs[seqAPI.SeqDBID]; !ok { +// return fmt.Errorf("unknown handlers.seq_api.seq_db_id %q", seqAPI.SeqDBID) +// } +// } +// +// return nil +// } + +// func validateAdmin(admin *Admin, cache *Cache) error { +// if admin == nil { +// return nil +// } +// +// if admin.Options.Cache != nil { +// if admin.Options.Cache.ID == "" { +// return fmt.Errorf("handlers.admin.options.cache.id cannot be empty") +// } +// redisCfg := cache.RedisByID(admin.Options.Cache.ID) +// if redisCfg == nil { +// return fmt.Errorf("unknown handlers.admin.options.cache.id %q", admin.Options.Cache.ID) +// } +// if redisCfg == nil { +// return fmt.Errorf("unknown handlers.admin.options.cache.id %q", admin.Options.Cache.ID) +// } +// if redisCfg.WithInmemID != "" { +// return fmt.Errorf("handlers.admin.options.cache.id %q: with_inmem_id is not allowed", admin.Options.Cache.ID) +// } +// if admin.Options.Cache.TTL <= 0 { +// admin.Options.Cache.TTL = defaultAdminCacheTTL +// } +// } +// +// if len(admin.Envs) > 0 { +// +// } +// } +// +// func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { +// if options.Limits.MaxAggregationsPerRequest <= 0 { +// options.Limits.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest +// } +// if options.Limits.MaxBucketsPerAggregationTs <= 0 { +// options.Limits.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs +// } +// if options.Limits.MaxParallelExportRequests <= 0 { +// options.Limits.MaxParallelExportRequests = defaultMaxParallelExportRequests +// } +// if options.Limits.MaxSearchTotal <= 0 { +// options.Limits.MaxSearchTotal = defaultMaxSearchTotal +// } +// if options.Limits.MaxSearchOffset <= 0 { +// options.Limits.MaxSearchOffset = defaultMaxSearchOffset +// } +// if options.Limits.MaxExportLimit <= 0 { +// options.Limits.MaxExportLimit = defaultMaxExportLimit +// } +// if options.Limits.MaxSearchLimit <= 0 { +// options.Limits.MaxSearchLimit = defaultMaxSearchLimit +// } +// if options.Limits.SeqCLIMaxSearchLimit <= 0 { +// options.Limits.SeqCLIMaxSearchLimit = defaultSeqCLIMaxSearchLimit +// } +// if options.Caches.Events.TTL <= 0 { +// options.Caches.Events.TTL = defaultEventsCacheTTL +// } +// if options.Caches.LogsLifespan.TTL <= 0 { +// options.Caches.LogsLifespan.TTL = defaultLogsLifespanCacheTTL +// } +// } diff --git a/internal/app/mw/auth.go b/internal/app/mw/auth.go index 4c3c486..0e4ed32 100644 --- a/internal/app/mw/auth.go +++ b/internal/app/mw/auth.go @@ -8,6 +8,7 @@ import ( "github.com/ozontech/seq-ui/internal/app/auth" "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" ) @@ -29,26 +30,32 @@ type AuthProviders struct { OidcProvider auth.OIDCProvider } -func NewAuthProviders( - ctx context.Context, - jwtSecretKey string, oidcCfg *config.OIDC, - oidcCache cache.Cache, -) (AuthProviders, error) { - authPrvds := AuthProviders{} +func (p *AuthProviders) enabled() bool { + return p.JwtProvider != nil || p.OidcProvider != nil +} + +// AuthProvidersByEnv contains auth providers by env, empty env is root options. +type AuthProvidersByEnv map[string]AuthProviders - if jwtSecretKey == "" && oidcCfg == nil { - return authPrvds, nil +func (providers AuthProvidersByEnv) get(ctx context.Context) AuthProviders { + if provider, ok := providers[types.GetEnvKey(ctx)]; ok { + return provider } + return providers[""] +} + +func NewAuthProvider(ctx context.Context, options config.AuthOptions, caches map[string]cache.Cache) (AuthProviders, error) { + authPrvds := AuthProviders{} - if jwtSecretKey != "" { + if options.JWT != nil { logger.Info("initializing jwt provider") - authPrvds.JwtProvider = auth.NewJWTProvider(jwtSecretKey) + authPrvds.JwtProvider = auth.NewJWTProvider(options.JWT.SecretKey) } - if oidcCfg != nil { + if options.OIDC != nil { logger.Info("initializing oidc provider") var err error - authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, oidcCfg, oidcCache) + authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, options.OIDC, caches[options.OIDC.CacheID]) if err != nil { return authPrvds, fmt.Errorf("failed to init oidc provider: %w", err) } diff --git a/internal/app/mw/env.go b/internal/app/mw/env.go new file mode 100644 index 0000000..1f15ba8 --- /dev/null +++ b/internal/app/mw/env.go @@ -0,0 +1,69 @@ +package mw + +import ( + "fmt" + + "github.com/ozontech/seq-ui/internal/app/config/v2" +) + +// HandlerEnvs describes envs of the handler. +type handlerEnvs struct { + DefaultEnv string + Envs map[string]struct{} +} + +// EnvResolver resolves env of the request for auth and rate limiting by api name. +// APIs not present in the resolver (userprofile, dashboards) or handlers without envs +// always use root options (empty env), env from the request is ignored for them. +type EnvResolver map[string]handlerEnvs + +// newEnvResolver collects envs of handlers that support envs. +func NewEnvResolver(handlers *config.Handlers) EnvResolver { + r := EnvResolver{} + + if h := handlers.SeqAPI; h != nil { + envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + r["seqapi"], r["SeqAPIService"] = envs, envs + } + if h := handlers.ErrorGroups; h != nil { + envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + r["errorgroups"], r["ErrorGroupsService"] = envs, envs + } + if h := handlers.MassExport; h != nil { + envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + r["massexport"], r["MassExportService"] = envs, envs + } + if h := handlers.Admin; h != nil { + envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + r["admin"], r["AdminService"] = envs, envs + } + if h := handlers.AsyncSearch; h != nil { + r["async_search"] = handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + } + return r +} + +func (resolver EnvResolver) resolve(api, env string) (string, error) { + handler, ok := resolver[api] + if !ok || len(handler.Envs) == 0 { + return "", nil + } + + if env == "" { + return handler.DefaultEnv, nil + } + + if _, ok := handler.Envs[env]; !ok { + return "", fmt.Errorf("env %q not found", env) + } + + return env, nil +} + +func envNames[T any](envs map[string]T) map[string]struct{} { + names := make(map[string]struct{}, len(envs)) + for name := range envs { + names[name] = struct{}{} + } + return names +} diff --git a/internal/app/mw/grpc_mw.go b/internal/app/mw/grpc_mw.go index a76e6f2..55b6b4d 100644 --- a/internal/app/mw/grpc_mw.go +++ b/internal/app/mw/grpc_mw.go @@ -3,6 +3,7 @@ package mw import ( "context" "fmt" + "strings" "time" "go.opentelemetry.io/otel/attribute" @@ -138,7 +139,7 @@ var noAuthGRPCMethods = map[string]map[string]struct{}{ var errUnauth = status.Error(codes.Unauthenticated, types.ErrUnauthenticated.Error()) -func GRPCAuthInterceptor(providers *AuthProviders) grpc.UnaryServerInterceptor { +func GRPCAuthInterceptor(providers *AuthProvidersByEnv) grpc.UnaryServerInterceptor { return func( ctx context.Context, req any, info *grpc.UnaryServerInfo, h grpc.UnaryHandler, @@ -151,6 +152,10 @@ func GRPCAuthInterceptor(providers *AuthProviders) grpc.UnaryServerInterceptor { if _, noAuth := noAuthGRPCMethods[svc][method]; noAuth { return h(ctx, req) } + provider := providers.get(ctx) + if !provider.enabled() { + return h(ctx, req) + } md, ok := metadata.FromIncomingContext(ctx) if !ok { logger.Error("no metadata provided") @@ -161,7 +166,7 @@ func GRPCAuthInterceptor(providers *AuthProviders) grpc.UnaryServerInterceptor { logger.Error("no authorization metadata provided") return nil, errUnauth } - userName, err := providers.auth(ctx, authHeaderSlice[0]) + userName, err := provider.auth(ctx, authHeaderSlice[0]) if err != nil { logger.Error("token auth failed", zap.Error(err)) return nil, errUnauth @@ -189,7 +194,7 @@ func GRPCRecoverInterceptor() grpc.UnaryServerInterceptor { } } -func GRPCRateLimitInterceptor(rateLimiters map[string]map[string]RateLimiter) grpc.UnaryServerInterceptor { +func GRPCRateLimitInterceptor(rateLimiters RateLimitersByEnv) grpc.UnaryServerInterceptor { return func( ctx context.Context, req any, info *grpc.UnaryServerInfo, h grpc.UnaryHandler, @@ -206,7 +211,7 @@ func GRPCRateLimitInterceptor(rateLimiters map[string]map[string]RateLimiter) gr return h(ctx, req) } - userToRateLimiter, ok := rateLimiters[svc] + userToRateLimiter, ok := rateLimiters.get(ctx)[svc] if !ok { return h(ctx, req) } @@ -244,3 +249,37 @@ func GRPCProcessHeadersInterceptor() grpc.UnaryServerInterceptor { return h(ctx, req) } } + +func GRPCEnvInterceptor(resolver EnvResolver) grpc.UnaryServerInterceptor { + return func( + ctx context.Context, req any, info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, + ) (resp any, err error) { + svc, method, err := parseGRPCFullMethod(info.FullMethod) + if err != nil { + msg := "failed to parse gRPC FullMethod" + logger.Error(msg, zap.Error(err)) + return nil, status.Error(codes.Internal, msg) + } + + var reqEnv string + if md, ok := metadata.FromIncomingContext(ctx); ok { + if values := md.Get("env"); len(values) > 0 { + reqEnv = values[0] + } + } + + api := svc + // async search methods are part of SeqAPIService, but have own envs + if svc == "SeqAPIService" && strings.Contains(method, "AsyncSearch") { + api = "async_search" + } + + env, err := resolver.resolve(api, reqEnv) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + return handler(types.SetEnvKey(ctx, env), req) + } +} diff --git a/internal/app/mw/http_mw.go b/internal/app/mw/http_mw.go index 7d4e1e6..daab4c3 100644 --- a/internal/app/mw/http_mw.go +++ b/internal/app/mw/http_mw.go @@ -142,7 +142,7 @@ func HTTPTraceInterceptor() func(next http.Handler) http.Handler { } } -func HTTPAuthInterceptor(providers *AuthProviders) func(next http.Handler) http.Handler { +func HTTPAuthInterceptor(providers AuthProvidersByEnv) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -157,8 +157,14 @@ func HTTPAuthInterceptor(providers *AuthProviders) func(next http.Handler) http. return } + provider := providers.get(ctx) + if !provider.enabled() { + next.ServeHTTP(w, r) + return + } + authHeader := r.Header.Get("Authorization") - username, err := providers.auth(ctx, authHeader) + username, err := provider.auth(ctx, authHeader) if err != nil { logger.Error("token auth failed", zap.Error(err)) http.Error(w, "Unauthenticated", http.StatusUnauthorized) @@ -191,7 +197,7 @@ func HTTPRecoverInterceptor() func(next http.Handler) http.Handler { } } -func HTTPRateLimitInterceptor(rateLimiters map[string]map[string]RateLimiter) func(next http.Handler) http.Handler { +func HTTPRateLimitInterceptor(rateLimiters RateLimitersByEnv) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -213,7 +219,7 @@ func HTTPRateLimitInterceptor(rateLimiters map[string]map[string]RateLimiter) fu http.NotFound(w, r) return } - userToRateLimiter, ok := rateLimiters[uriApi] + userToRateLimiter, ok := rateLimiters.get(ctx)[uriApi] if !ok { next.ServeHTTP(w, r) return @@ -271,3 +277,29 @@ func HTTPProcessHeadersInterceptor() func(next http.Handler) http.Handler { return http.HandlerFunc(fn) } } + +func HTTPEnvInterceptor(resolver EnvResolver) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + uriApi, _, uriMethod, err := parseURI(r.RequestURI) + if err != nil { + next.ServeHTTP(w, r) + return + } + + // async ssearch routes are part of seqapi, but have own envs + if uriApi == "seqapi" && uriMethod == "async_search" { + uriApi = "async_search" + } + + env, err := resolver.resolve(uriApi, r.URL.Query().Get("env")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + next.ServeHTTP(w, r.WithContext(types.SetEnvKey(r.Context(), env))) + } + return http.HandlerFunc(fn) + } +} diff --git a/internal/app/mw/ratelimiter.go b/internal/app/mw/ratelimiter.go index 07cf642..adaa6c5 100644 --- a/internal/app/mw/ratelimiter.go +++ b/internal/app/mw/ratelimiter.go @@ -33,7 +33,41 @@ type RateLimiter struct { perHandler bool } -func NewRateLimiter(api string, cfg config.RateLimiter) (RateLimiter, error) { +// RateLimitersByEnv contains rate limiters by env->api->user, empty env is root options. +type RateLimitersByEnv map[string]map[string]map[string]RateLimiter + +func (ratelimiters RateLimitersByEnv) get(ctx context.Context) map[string]map[string]RateLimiter { + if ratelimiter, ok := ratelimiters[types.GetEnvKey(ctx)]; ok { + return ratelimiter + } + return ratelimiters[""] +} + +func NewRateLimiters(apiToRateLimiters config.ApiToRateLimiters) (map[string]map[string]RateLimiter, error) { + result := make(map[string]map[string]RateLimiter, len(apiToRateLimiters)) + + for apiName, rateLimiters := range apiToRateLimiters { + result[apiName] = make(map[string]RateLimiter) + + defaultLimiter, err := newRateLimiter(apiName, rateLimiters.Default) + if err != nil { + return nil, fmt.Errorf("init default rate limiter: %w", err) + } + result[apiName][RateLimiterDefaultUser] = defaultLimiter + + for user, rateLimiter := range rateLimiters.SpecialUsers { + limiter, err := newRateLimiter(apiName, rateLimiter) + if err != nil { + return nil, fmt.Errorf("init user %q rate limiter: %w", user, err) + } + result[apiName][user] = limiter + } + } + + return result, nil +} + +func newRateLimiter(api string, cfg config.RateLimiter) (RateLimiter, error) { if _, ok := grpcServices[api]; !ok { spec := swagger.GetSpecEx() if !spec.HasApi(api) { diff --git a/internal/app/server/init.go b/internal/app/server/init.go index ab760da..781eb88 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -5,12 +5,12 @@ import ( "fmt" "net" "net/http" + "time" "github.com/go-chi/chi/v5" "github.com/go-chi/cors" grpc_mw "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/prometheus/client_golang/prometheus/promhttp" - "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -21,60 +21,54 @@ import ( "github.com/ozontech/seq-ui/tracing" ) -const ( - defaultCORSAllowedOrigins = "*" - maxHTTPHeaderBytes = 1 << 12 // 4 KiB -) - -var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE"} - func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { - var err error - var jwtSecretKey string - var oidcCfg *config.OIDC - - if s.config.Auth != nil { - if s.config.Auth.JWT != nil { - jwtSecretKey = s.config.Auth.JWT.SecretKey - } - oidcCfg = s.config.Auth.OIDC + if err := s.prepareAuth(ctx); err != nil { + return err } - - s.authPrvds, err = mw.NewAuthProviders(ctx, jwtSecretKey, oidcCfg, s.oidcCache) - if err != nil { + if err := s.prepareRateLimiters(); err != nil { return err } - - err = s.prepareRateLimiters() - if err != nil { + if s.config.GRPC != nil { + s.prepareGRPCServer(registrar) + } + if s.config.GRPC != nil { + s.prepareHTTPServer(ctx, registrar) + } + if err := s.prepareDebugServer(ctx); err != nil { return err } - s.prepareGRPCServer(registrar) + return nil +} + +func (s *Server) prepareAuth(ctx context.Context) error { + if s.config.Auth == nil { + return nil + } - s.prepareHTTPServer(ctx, registrar) + s.authPrvds = make(mw.AuthProvidersByEnv, len(s.config.Auth.Envs)+1) - err = s.prepareDebugServer(ctx) + provider, err := mw.NewAuthProvider(ctx, s.config.Auth.Options, s.caches) if err != nil { - return err + return fmt.Errorf("init auth: %w", err) } + s.authPrvds[""] = provider + for envName, envConfig := range s.config.Auth.Envs { + provider, err := mw.NewAuthProvider(ctx, envConfig, s.caches) + if err != nil { + return fmt.Errorf("init auth for env %q: %w", envName, err) + } + s.authPrvds[envName] = provider + } return nil } // setupCORS applies CORS policies set in config to the provided mux. func (s *Server) setupCORS(mux *chi.Mux) { - allowedOrigins := s.config.HTTP.CORS.AllowedOrigins - if len(allowedOrigins) == 0 { - allowedOrigins = append(allowedOrigins, defaultCORSAllowedOrigins) - } - allowedMethods := s.config.HTTP.CORS.AllowedMethods - if len(allowedMethods) == 0 { - allowedMethods = append(allowedMethods, defaultCORSAllowedMethods...) - } mux.Use(cors.Handler(cors.Options{ - AllowedOrigins: allowedOrigins, - AllowedMethods: allowedMethods, + AllowedOrigins: s.config.HTTP.CORS.AllowedOrigins, + AllowedMethods: s.config.HTTP.CORS.AllowedMethods, AllowedHeaders: s.config.HTTP.CORS.AllowedHeaders, ExposedHeaders: s.config.HTTP.CORS.ExposedHeaders, AllowCredentials: s.config.HTTP.CORS.AllowCredentials, @@ -85,36 +79,24 @@ func (s *Server) setupCORS(mux *chi.Mux) { // prepareRateLimiters prepares requests rate limiters based on server config. func (s *Server) prepareRateLimiters() error { - if len(s.config.RateLimiters) == 0 { + if s.config.RateLimiting == nil { return nil } - s.rateLimiters = make(map[string]map[string]mw.RateLimiter) + s.rateLimiters = make(mw.RateLimitersByEnv, len(s.config.RateLimiting.Envs)+1) - for apiName, rateLimiters := range s.config.RateLimiters { - s.rateLimiters[apiName] = make(map[string]mw.RateLimiter) + limiters, err := mw.NewRateLimiters(s.config.RateLimiting.Options) + if err != nil { + return fmt.Errorf("init rate limiter: %w", err) + } + s.rateLimiters[""] = limiters - logger.Info("init default rate limiter", zap.String("api", apiName)) - defaultLimiter, err := mw.NewRateLimiter(apiName, rateLimiters.Default) + for envName, envConfig := range s.config.RateLimiting.Envs { + limiters, err := mw.NewRateLimiters(envConfig) if err != nil { - return fmt.Errorf("init default rate limiter: %w", err) - } - - s.rateLimiters[apiName][mw.RateLimiterDefaultUser] = defaultLimiter - - for user, rateLimiter := range rateLimiters.SpecialUsers { - logger.Info( - "init user rate limiter", - zap.String("api", apiName), - zap.String("user", user), - ) - limiter, err := mw.NewRateLimiter(apiName, rateLimiter) - if err != nil { - return fmt.Errorf("init user %q rate limiter: %w", user, err) - } - - s.rateLimiters[apiName][user] = limiter + return fmt.Errorf("init rate limiter for env %q: %w", envName, err) } + s.rateLimiters[envName] = limiters } return nil @@ -125,8 +107,9 @@ func (s *Server) prepareGRPCServer(registrar *api.Registrar) { interceptors := []grpc.UnaryServerInterceptor{ mw.GRPCRecoverInterceptor(), mw.GRPCProcessHeadersInterceptor(), + mw.GRPCEnvInterceptor(s.envResolver), } - if s.authPrvds.JwtProvider != nil || s.authPrvds.OidcProvider != nil { + if len(s.authPrvds) > 0 { interceptors = append(interceptors, mw.GRPCAuthInterceptor(&s.authPrvds)) } interceptors = append(interceptors, @@ -160,9 +143,10 @@ func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar mw.HTTPNotFoundInterceptor(), mw.HTTPRecoverInterceptor(), mw.HTTPProcessHeadersInterceptor(), + mw.HTTPEnvInterceptor(s.envResolver), } - if s.authPrvds.JwtProvider != nil || s.authPrvds.OidcProvider != nil { - interceptors = append(interceptors, mw.HTTPAuthInterceptor(&s.authPrvds)) + if len(s.authPrvds) > 0 { + interceptors = append(interceptors, mw.HTTPAuthInterceptor(s.authPrvds)) } interceptors = append(interceptors, mw.HTTPMetricInterceptor(), @@ -176,13 +160,13 @@ func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar registrar.RegisterHTTPHandlers(mux) - s.httpServer = s.makeHTTPServer(ctx, mux) + s.httpServer = s.makeHTTPServer(ctx, mux, s.config.HTTP.ReadHeaderTimeout, s.config.HTTP.ReadTimeout, s.config.HTTP.WriteTimeout) } // prepareHTTPMux prepares debug HTTP server mux with applied CORS policies and added interceptors. func (s *Server) prepareDebugServer(ctx context.Context) error { mux := chi.NewMux() - if s.config.HTTP.CORS != nil { + if s.config.HTTP != nil && s.config.HTTP.CORS != nil { s.setupCORS(mux) } interceptors := chi.Middlewares{ @@ -191,24 +175,27 @@ func (s *Server) prepareDebugServer(ctx context.Context) error { mux.Use(interceptors...) mux.Handle("/metrics", promhttp.Handler()) serveHealth(mux) - _, port, err := net.SplitHostPort(s.config.HTTP.Addr) - if err != nil { - return err + // swagger describes http api, so it is served only if http server is configured + if s.config.HTTP != nil { + _, port, err := net.SplitHostPort(s.config.HTTP.Addr) + if err != nil { + return err + } + serveSwaggerUI(mux, port) } - serveSwaggerUI(mux, port) servePprof(mux) - s.debugServer = s.makeHTTPServer(ctx, mux) + s.debugServer = s.makeHTTPServer(ctx, mux, s.config.Debug.ReadHeaderTimeout, s.config.Debug.ReadTimeout, s.config.Debug.WriteTimeout) return nil } // makeHTTPServer makes HTTP server from provided mux. -func (s *Server) makeHTTPServer(ctx context.Context, mux *chi.Mux) *http.Server { +func (s *Server) makeHTTPServer(ctx context.Context, mux *chi.Mux, readHeaderTimeout, readTimeout, writeTimeout time.Duration) *http.Server { return &http.Server{ Handler: mux, - ReadHeaderTimeout: s.config.HTTP.ReadHeaderTimeout, - ReadTimeout: s.config.HTTP.ReadTimeout, - WriteTimeout: s.config.HTTP.WriteTimeout, - MaxHeaderBytes: maxHTTPHeaderBytes, // 4 KiB + ReadHeaderTimeout: readHeaderTimeout, + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + MaxHeaderBytes: config.MaxHTTPHeaderBytes, // 4 KiB BaseContext: func(_ net.Listener) context.Context { return ctx }, diff --git a/internal/app/server/run.go b/internal/app/server/run.go index a7346ba..78c9e78 100644 --- a/internal/app/server/run.go +++ b/internal/app/server/run.go @@ -17,24 +17,28 @@ func (s *Server) Run(ctx context.Context) error { errWg, ctx := errgroup.WithContext(ctx) // run gRPC server - errWg.Go(func() error { - lis, err := net.Listen("tcp", s.config.GRPC.Addr) - if err != nil { - return err - } - - return s.grpcServer.Serve(lis) - }) + if s.grpcServer != nil { + errWg.Go(func() error { + lis, err := net.Listen("tcp", s.config.GRPC.Addr) + if err != nil { + return err + } + + return s.grpcServer.Serve(lis) + }) + } // run HTTP server - errWg.Go(func() error { - l, err := net.Listen("tcp", s.config.HTTP.Addr) - if err != nil { - return err - } - - return s.httpServer.Serve(l) - }) + if s.httpServer != nil { + errWg.Go(func() error { + l, err := net.Listen("tcp", s.config.HTTP.Addr) + if err != nil { + return err + } + + return s.httpServer.Serve(l) + }) + } // run debug server errWg.Go(func() error { @@ -47,13 +51,18 @@ func (s *Server) Run(ctx context.Context) error { }) errWg.Go(func() error { - swaggerAddr := s.config.Debug.Addr + swaggerUIPrefix - logger.Info("app started", - zap.String("http", s.config.HTTP.Addr), - zap.String("debug", s.config.Debug.Addr), - zap.String("swagger", swaggerAddr), - zap.String("grpc", s.config.GRPC.Addr), - ) + fields := []zap.Field{zap.String("debug", s.config.Debug.Addr)} + if s.httpServer != nil { + fields = append(fields, + zap.String("http", s.config.HTTP.Addr), + zap.String("swagger", s.config.Debug.Addr+swaggerUIPrefix), + ) + } + if s.grpcServer != nil { + fields = append(fields, zap.String("grpc", s.config.GRPC.Addr)) + } + + logger.Info("app started", fields...) return nil }) @@ -71,43 +80,42 @@ func (s *Server) Run(ctx context.Context) error { // Stop the gRPC and HTTP servers. func (s *Server) stop() { - var wg sync.WaitGroup - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - wg.Add(1) - go func() { - if err := s.httpServer.Shutdown(ctx); err != nil { - logger.Error("shutting down the http server", zap.Error(err)) - } else { - logger.Warn("http server gracefully stopped") - } - wg.Done() - }() + var wg sync.WaitGroup + if s.httpServer != nil { + wg.Go(func() { + if err := s.httpServer.Shutdown(ctx); err != nil { + logger.Error("shutting down the http server", zap.Error(err)) + } else { + logger.Warn("http server gracefully stopped") + } + }) + } - wg.Add(1) - go func() { + wg.Go(func() { if err := s.debugServer.Shutdown(ctx); err != nil { logger.Error("shutting down the debug server", zap.Error(err)) } else { logger.Warn("debug server gracefully stopped") } - wg.Done() - }() - - done := make(chan struct{}) - go func() { - s.grpcServer.GracefulStop() - close(done) - }() - select { - case <-done: - logger.Warn("grpc server gracefully stopped") - case <-ctx.Done(): - logger.Error("shutting down the grpc server", zap.Error(ctx.Err())) - logger.Warn("grpc server force stop") - s.grpcServer.Stop() + }) + + if s.grpcServer != nil { + done := make(chan struct{}) + go func() { + s.grpcServer.GracefulStop() + close(done) + }() + select { + case <-done: + logger.Warn("grpc server gracefully stopped") + case <-ctx.Done(): + logger.Error("shutting down the grpc server", zap.Error(ctx.Err())) + logger.Warn("grpc server force stop") + s.grpcServer.Stop() + } } wg.Wait() } diff --git a/internal/app/server/server.go b/internal/app/server/server.go index 81afc0a..37f09be 100644 --- a/internal/app/server/server.go +++ b/internal/app/server/server.go @@ -16,18 +16,23 @@ import ( // Server contains application dependencies. type Server struct { config *config.Server - oidcCache cache.Cache debugServer *http.Server grpcServer *grpc.Server httpServer *http.Server + caches map[string]cache.Cache - authPrvds mw.AuthProviders - rateLimiters map[string]map[string]mw.RateLimiter // rate limiter by api and user + envResolver mw.EnvResolver + authPrvds mw.AuthProvidersByEnv + rateLimiters mw.RateLimitersByEnv } // New returns a new Server. -func New(ctx context.Context, cfg *config.Server, registrar *api.Registrar, oidcCache cache.Cache) (*Server, error) { - s := &Server{config: cfg, oidcCache: oidcCache} +func New(ctx context.Context, cfg *config.Config, registrar *api.Registrar, caches map[string]cache.Cache) (*Server, error) { + s := &Server{ + config: &cfg.Server, + caches: caches, + envResolver: mw.NewEnvResolver(&cfg.Handlers), + } if err := s.init(ctx, registrar); err != nil { return nil, fmt.Errorf("init server: %w", err) diff --git a/internal/app/server/tech.go b/internal/app/server/tech.go index 4e612d9..954684b 100644 --- a/internal/app/server/tech.go +++ b/internal/app/server/tech.go @@ -15,6 +15,9 @@ import ( "github.com/ozontech/seq-ui/swagger" ) +const swaggerUIPrefix = "/docs/" +const swaggerJSONPath = "/swagger.json" + func serveHealth(mux *chi.Mux) { mux.HandleFunc("/live", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("{}")) @@ -24,9 +27,6 @@ func serveHealth(mux *chi.Mux) { }) } -const swaggerUIPrefix = "/docs/" -const swaggerJSONPath = "/swagger.json" - func defSwagger(host string) ([]byte, error) { s := swagger.GetSpec() s.Host = host @@ -55,6 +55,8 @@ func serveSwaggerUI(mux *chi.Mux, apiPort string) { def, err := defSwagger(r.Host) if err != nil { logger.Error("failed to get swagger def", zap.Error(err)) + http.Error(w, "failed to get swagger def", http.StatusInternalServerError) + return } w.Header().Set("Content-Type", "application/json") diff --git a/internal/app/types/ctx.go b/internal/app/types/ctx.go index b07e972..5570c5c 100644 --- a/internal/app/types/ctx.go +++ b/internal/app/types/ctx.go @@ -38,3 +38,18 @@ func GetUseSeqQL(ctx context.Context) string { } return useSeqQL } + +type EnvKey struct{} + +// SetEnvKey returns a new context with the env. +func SetEnvKey(ctx context.Context, env string) context.Context { + return context.WithValue(ctx, EnvKey{}, env) +} + +// GetEnvKey returns env from context. +func GetEnvKey(ctx context.Context) string { + if v := ctx.Value(EnvKey{}); v != nil { + return v.(string) + } + return "" +} diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index e8b5a50..87da6d6 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -15,7 +15,7 @@ func FromConfig(ctx context.Context, cfg *config.Cache) (map[string]Cache, error for i := range cfg.Redis { redis := &cfg.Redis[i] - var inmem *config.InmemoryCache + var inmem *config.Inmemory if redis.WithInmemID != "" { inmem = cfg.InmemByID(redis.WithInmemID) } @@ -45,7 +45,7 @@ func FromConfig(ctx context.Context, cfg *config.Cache) (map[string]Cache, error return caches, nil } -func New(ctx context.Context, inmemCfg *config.InmemoryCache, redisCfg *config.Redis) (Cache, error) { +func New(ctx context.Context, inmemCfg *config.Inmemory, redisCfg *config.Redis) (Cache, error) { redis, redisErr := newRedisCache(ctx, redisCfg) if inmemCfg == nil { if redisErr != nil { diff --git a/internal/pkg/cache/inmemory.go b/internal/pkg/cache/inmemory.go index 5910545..8250f02 100644 --- a/internal/pkg/cache/inmemory.go +++ b/internal/pkg/cache/inmemory.go @@ -17,7 +17,7 @@ type inmemoryCache struct { cache *ristretto.Cache[string, string] } -func newInmemoryCache(cfg config.InmemoryCache) (*inmemoryCache, error) { +func newInmemoryCache(cfg config.Inmemory) (*inmemoryCache, error) { c, err := ristretto.NewCache(&ristretto.Config[string, string]{ NumCounters: cfg.NumCounters, MaxCost: cfg.MaxCost, diff --git a/internal/pkg/service/massexport/filestore/s3.go b/internal/pkg/service/massexport/filestore/s3.go index d082a45..3b5309e 100644 --- a/internal/pkg/service/massexport/filestore/s3.go +++ b/internal/pkg/service/massexport/filestore/s3.go @@ -22,7 +22,7 @@ type s3FileStore struct { func NewS3(ctx context.Context, cfg *config.S3) (FileStore, error) { scheme := "http" - if cfg.EnableSSl { + if cfg.EnableSSL { scheme = "https" } From cac00da5a8c999fef24f61ee80052f4a64a07b0b Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 14 Oct 2026 01:22:59 +0300 Subject: [PATCH 19/29] update server/mw/config/auth/cmd --- cmd/seq-ui/main.go | 104 +++++++++++++-------------- internal/app/auth/jwt.go | 6 +- internal/app/auth/jwt_test.go | 6 +- internal/app/auth/oidc.go | 13 +++- internal/app/config/loader/loader.go | 22 +++--- internal/app/config/loader/merge.go | 24 +++++-- internal/app/config/migrate/v1_v2.go | 1 + internal/app/config/v2/config.go | 33 +++++---- internal/app/config/v2/validate.go | 2 +- internal/app/mw/auth.go | 7 +- internal/app/mw/grpc_mw.go | 2 +- internal/app/mw/http_mw.go | 5 +- internal/app/mw/ratelimiter.go | 9 +-- internal/app/server/init.go | 53 ++++++++------ internal/app/server/server.go | 5 +- internal/pkg/cache/ctor.go | 66 +++++++---------- internal/pkg/cache/redis.go | 4 +- 17 files changed, 190 insertions(+), 172 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index bcb66e2..bf6835a 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -21,7 +21,6 @@ import ( admin_v1 "github.com/ozontech/seq-ui/internal/api/admin/v1" dashboards_v1 "github.com/ozontech/seq-ui/internal/api/dashboards/v1" errorgroups_v1 "github.com/ozontech/seq-ui/internal/api/errorgroups/v1" - massexport_v1 "github.com/ozontech/seq-ui/internal/api/massexport/v1" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" "github.com/ozontech/seq-ui/internal/app/config/loader" @@ -30,11 +29,9 @@ import ( "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/repository" - repositorych "github.com/ozontech/seq-ui/internal/pkg/repository_ch" "github.com/ozontech/seq-ui/internal/pkg/service/admin" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" "github.com/ozontech/seq-ui/internal/pkg/service/dashboards" - "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" "github.com/ozontech/seq-ui/internal/pkg/service/massexport" "github.com/ozontech/seq-ui/internal/pkg/service/massexport/filestore" "github.com/ozontech/seq-ui/internal/pkg/service/massexport/sessionstore" @@ -94,14 +91,13 @@ func run(ctx context.Context) { } logger.Info("initializing caches") - caches, err := cache.FromConfig(ctx, &cfg.Cache) - if err != nil { + if err := cache.Init(ctx, &cfg.Cache); err != nil { logger.Fatal("failed to init caches", zap.Error(err)) } - registrar := initApp(ctx, cfg, caches) + registrar := initApp(ctx, cfg) - serv, err := server.New(ctx, &cfg, registrar, caches) + serv, err := server.New(ctx, cfg, registrar) if err != nil { logger.Fatal("app init error", zap.Error(err)) } @@ -113,7 +109,7 @@ func run(ctx context.Context) { } } -func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cache) *api.Registrar { +func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { @@ -132,17 +128,16 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac logger.Fatal("failed to init db", zap.Error(err)) } - var massExportV1 *massexport_v1.MassExport - if cfg.Handlers.MassExport != nil { - me := cfg.Handlers.MassExport - redisCfg := cfg.Cache.RedisByID(me.SessionStore.RedisID) - - exportServer, err := initExportService(ctx, *me, redisCfg, seqDBClients[me.SeqDBID]) - if err != nil { - logger.Fatal("can't init export server", zap.Error(err)) - } - massExportV1 = massexport_v1.New(exportServer) - } + // Это ужас пока оставим, на создание handlers не смотри пока + // var massExportV1 *massexport_v1.MassExport + // if cfg.Handlers.MassExport != nil { + // logger.Info("initializing mass export") + // exportServer, err := initExportService(ctx, cfg.Handlers.MassExport, seqDBClients) + // if err != nil { + // logger.Fatal("can't init mass export server", zap.Error(err)) + // } + // massExportV1 = massexport_v1.New(exportServer) + // } var ( adminV1 *admin_v1.Admin @@ -156,27 +151,24 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac dashboardsSvc := dashboards.New(repo.Dashboards) profiles.InitProfiles(repo.UserProfiles.GetOrCreate) + logger.Info("initializing user profile") userProfileV1 = userprofile_v1.New(userProfilesSvc) + logger.Info("initializing dashboards") dashboardsV1 = dashboards_v1.New(dashboardsSvc) if cfg.Handlers.AsyncSearch != nil { - asyncSearchesSvc = asyncsearches.New(ctx, repo, seqDBClients[cfg.Handlers.AsyncSearch.SeqDBID], *cfg.Handlers.AsyncSearch) + logger.Info("initializing async search") + asyncSearchesSvc = asyncsearches.New(ctx, repo, seqDBClients, *cfg.Handlers.AsyncSearch) } if cfg.Handlers.Admin != nil { - logger.Info("initializing redis admin cache") - - adminCache := caches[cfg.Handlers.Admin.RedisID] - if adminCache == nil { - logger.Info("redis cache config is not set, admin will run without cache") - } - + logger.Info("initializing admin") adminSvc := admin.New(repo.Admin, repo.UserProfiles, adminCache, cfg.Handlers.Admin) adminV1 = admin_v1.New(adminSvc) } } - seqAPICaches := cfg.Handlers.SeqAPI.Options.Caches + logger.Info("initializing seqapi") seqApiV1 := seqapi_v1.New( &cfg.Handlers.SeqAPI, seqDBClients, @@ -188,22 +180,19 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac var errorGroupsV1 *errorgroups_v1.ErrorGroups if cfg.Handlers.ErrorGroups != nil { - chCfg := cfg.Clients.ClickHouseByID(cfg.Handlers.ErrorGroups.CHID) - repo := repositorych.New(chClients[cfg.Handlers.ErrorGroups.CHID], chCfg.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) - svc := errorgroups.New(repo, cfg.Handlers.ErrorGroups.LogTagsMapping) - - errorGroupsV1 = errorgroups_v1.New(svc, chClients) + logger.Info("initializing error groups") + errorGroupsV1 = errorgroups_v1.New(cfg.Handlers.ErrorGroups, &cfg.Clients, chClients) } return api.NewRegistrar(adminV1, seqApiV1, userProfileV1, dashboardsV1, massExportV1, errorGroupsV1) } -func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb.Client, error) { +func initSeqDBClients(ctx context.Context, cfg *config.Config) (map[string]seqdb.Client, error) { clients := make(map[string]seqdb.Client, len(cfg.Clients.SeqDB)) for _, clientCfg := range cfg.Clients.SeqDB { - client, err := createSeqBDClient(ctx, &clientCfg, &cfg.Handlers.SeqAPI) + client, err := createSeqBDClient(ctx, &clientCfg, cfg.Handlers.SeqAPI) if err != nil { - return nil, fmt.Errorf("failed to create seq_db client %s: %w", clientCfg.ID, err) + return nil, fmt.Errorf("failed to create seq_db client %q: %w", clientCfg.ID, err) } clients[clientCfg.ID] = client } @@ -212,31 +201,38 @@ func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb. } func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *config.SeqAPI) (seqdb.Client, error) { - maxSearchLimit := int(seqAPI.Options.Limits.MaxSearchLimit) - for _, env := range seqAPI.Envs { - if env.SeqDBID != cfg.ID { - continue + var maxSearchLimit int + if seqAPI != nil { + if len(seqAPI.Envs) == 0 { + maxSearchLimit = seqAPI.Options.Limits.MaxSearchLimit } - if l := int(env.Options.Limits.MaxSearchLimit); l > maxSearchLimit { - maxSearchLimit = l + + // one seqdb client can be used by several envs, so it must accept the largest one + for _, env := range seqAPI.Envs { + if env.SeqDBID == cfg.ID { + maxSearchLimit = max(maxSearchLimit, env.Options.Limits.MaxSearchLimit) + } } } + clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * maxSearchLimit clientMaxRecvMsgSize = max(clientMaxRecvMsgSize, defaultClientMaxRecvMsgSize) clientParams := seqdb.ClientParams{ - Addrs: cfg.Addrs, - Timeout: cfg.Timeout, - MaxRetries: cfg.RequestRetries, - InitialRetryBackoff: cfg.InitialRetryBackoff, - MaxRetryBackoff: cfg.MaxRetryBackoff, - MaxRecvMsgSize: clientMaxRecvMsgSize, + Addrs: cfg.Addrs, + Timeout: cfg.Timeout, + MaxRetries: cfg.RequestRetries, + MaxRecvMsgSize: clientMaxRecvMsgSize, + } + if cfg.RequestRetriesBackoff != nil { + clientParams.InitialRetryBackoff = cfg.RequestRetriesBackoff.InitialInterval + clientParams.MaxRetryBackoff = cfg.RequestRetriesBackoff.MaxInterval } - if cfg.GRPCKeepaliveParams != nil { + if cfg.GRPCParams != nil && cfg.GRPCParams.Keepalive != nil { clientParams.GRPCKeepaliveParams = &seqdb.GRPCKeepaliveParams{ - Time: cfg.GRPCKeepaliveParams.Time, - Timeout: cfg.GRPCKeepaliveParams.Timeout, - PermitWithoutStream: cfg.GRPCKeepaliveParams.PermitWithoutStream, + Time: cfg.GRPCParams.Keepalive.Time, + Timeout: cfg.GRPCParams.Keepalive.Timeout, + PermitWithoutStream: cfg.GRPCParams.Keepalive.PermitWithoutStream, } } @@ -283,12 +279,12 @@ func initExportService(ctx context.Context, cfg config.MassExport, redisCfg *con return massexport.NewService(ctx, cfg, sessionStore, fileStore, client) } -func initClickHouseClients(ctx context.Context, cfg config.Config) (map[string]driver.Conn, error) { +func initClickHouseClients(ctx context.Context, cfg *config.Config) (map[string]driver.Conn, error) { clients := make(map[string]driver.Conn, len(cfg.Clients.ClickHouse)) for _, clientCfg := range cfg.Clients.ClickHouse { client, err := createClickHouseClient(ctx, &clientCfg) if err != nil { - return nil, fmt.Errorf("failed to create clickhouse client %s: %w", clientCfg.ID, err) + return nil, fmt.Errorf("failed to create clickhouse client %q: %w", clientCfg.ID, err) } clients[clientCfg.ID] = client } diff --git a/internal/app/auth/jwt.go b/internal/app/auth/jwt.go index c6dcbc6..988d4e1 100644 --- a/internal/app/auth/jwt.go +++ b/internal/app/auth/jwt.go @@ -6,6 +6,8 @@ import ( "time" "github.com/golang-jwt/jwt/v4" + + "github.com/ozontech/seq-ui/internal/app/config/v2" ) type JWTClaims struct { @@ -23,9 +25,9 @@ type jwtProvider struct { secretKey string } -func NewJWTProvider(secretKey string) JWTProvider { +func NewJWTProvider(cfg *config.JWT) JWTProvider { return &jwtProvider{ - secretKey: secretKey, + secretKey: cfg.SecretKey, } } diff --git a/internal/app/auth/jwt_test.go b/internal/app/auth/jwt_test.go index 684739b..542a453 100644 --- a/internal/app/auth/jwt_test.go +++ b/internal/app/auth/jwt_test.go @@ -5,12 +5,14 @@ import ( "time" "github.com/stretchr/testify/require" + + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestJWTProvider(t *testing.T) { expInterval := time.Second - p1 := NewJWTProvider("secret-1") - p2 := NewJWTProvider("secret-2") + p1 := NewJWTProvider(&config.JWT{SecretKey: "secret-1"}) + p2 := NewJWTProvider(&config.JWT{SecretKey: "secret-2"}) // test valid issue token tokenName1 := "test-1" testStart := time.Now() diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 002b3e7..3589f39 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -42,12 +42,19 @@ type oidcProvider struct { allowedClients []string } -func NewOIDCProvider(ctx context.Context, cfg *config.OIDC, oidcCache cache.Cache) (OIDCProvider, error) { +func NewOIDCProvider(ctx context.Context, cfg *config.OIDC) (OIDCProvider, error) { var ( - err error - oidcCtx context.Context + err error + oidcCtx context.Context + oidcCache cache.Cache ) + if cfg.CacheID != "" { + if oidcCache = cache.ByID(cfg.CacheID); oidcCache == nil { + return nil, fmt.Errorf("oidc cache %q not found", cfg.CacheID) + } + } + if !cfg.SkipVerify { oidcCtx, err = newHTTPContext(context.Background(), cfg.TLS) if err != nil { diff --git a/internal/app/config/loader/loader.go b/internal/app/config/loader/loader.go index 34f6971..9aac91a 100644 --- a/internal/app/config/loader/loader.go +++ b/internal/app/config/loader/loader.go @@ -25,47 +25,47 @@ type configMeta struct { } // FromFile parse config from config path. -func FromFile(cfgPath string) (v2.Config, error) { +func FromFile(cfgPath string) (*v2.Config, error) { cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec if err != nil { - return v2.Config{}, fmt.Errorf("read file: %w", err) + return nil, fmt.Errorf("read file: %w", err) } version, err := readVersion(cfgBytes) if err != nil { - return v2.Config{}, fmt.Errorf("read version: %w", err) + return nil, fmt.Errorf("read version: %w", err) } switch version { case V1: cfgV1, err := decode[v1.Config](cfgBytes, true) if err != nil { - return v2.Config{}, fmt.Errorf("parse config v1: %w", err) + return nil, fmt.Errorf("parse config v1: %w", err) } cfgBytes, err = encode(migrate.V1ToV2(cfgV1)) if err != nil { - return v2.Config{}, err + return nil, err } case V2: default: - return v2.Config{}, fmt.Errorf("unsupported config version: %d", version) + return nil, fmt.Errorf("unsupported config version: %d", version) } cfgBytes, err = mergeEnvOptions(cfgBytes) if err != nil { - return v2.Config{}, fmt.Errorf("merge env options: %w", err) + return nil, fmt.Errorf("merge env options: %w", err) } cfg, err := decode[v2.Config](cfgBytes, true) if err != nil { - return v2.Config{}, fmt.Errorf("parse config v2: %w", err) + return nil, fmt.Errorf("parse config v2: %w", err) } if err := cfg.Validate(); err != nil { - return v2.Config{}, fmt.Errorf("normalize config: %w", err) + return nil, fmt.Errorf("validate config: %w", err) } - return cfg, nil + return &cfg, nil } func ToLatestVersion(cfgBytes []byte) ([]byte, error) { @@ -113,7 +113,7 @@ func decode[T any](cfg []byte, strict bool) (T, error) { return result, nil } -func encode(cfg v2.Config) ([]byte, error) { +func encode[T any](cfg T) ([]byte, error) { var buf bytes.Buffer encoder := yaml.NewEncoder(&buf) encoder.SetIndent(2) diff --git a/internal/app/config/loader/merge.go b/internal/app/config/loader/merge.go index c05b866..0735768 100644 --- a/internal/app/config/loader/merge.go +++ b/internal/app/config/loader/merge.go @@ -13,7 +13,7 @@ func mergeEnvOptions(cfgBytes []byte) ([]byte, error) { return nil, fmt.Errorf("parse yaml for merge: %w", err) } - merge := func(root map[string]any) { + mergeHandlerOptions := func(root map[string]any) { rootOpts, _ := root["options"].(map[string]any) envs, _ := root["envs"].(map[string]any) if rootOpts == nil || len(envs) == 0 { @@ -31,21 +31,33 @@ func mergeEnvOptions(cfgBytes []byte) ([]byte, error) { } } + mergeServerOptions := func(root map[string]any) { + rootOpts, _ := root["options"].(map[string]any) + envs, _ := root["envs"].(map[string]any) + if rootOpts == nil || len(envs) == 0 { + return + } + + for name, e := range envs { + envOpts, _ := e.(map[string]any) + envs[name] = mergeYAMLs(rootOpts, envOpts) + } + } + handlers, _ := root["handlers"].(map[string]any) for _, h := range handlers { if handler, _ := h.(map[string]any); handler != nil { - merge(handler) + mergeHandlerOptions(handler) } } server, _ := root["server"].(map[string]any) if auth, _ := server["auth"].(map[string]any); auth != nil { - merge(auth) + mergeServerOptions(auth) } - if rl, _ := server["rate_limiters"].(map[string]any); rl != nil { - merge(rl) + if rl, _ := server["rate_limiting"].(map[string]any); rl != nil { + mergeServerOptions(rl) } - out, err := yaml.Marshal(root) if err != nil { return nil, fmt.Errorf("marshal merged yaml: %w", err) diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index db612a9..b54b95e 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -53,6 +53,7 @@ func (m *v1tov2) migrateServer(src *v1.Server) v2.Server { ReadTimeout: src.HTTPReadTimeout, ReadHeaderTimeout: src.HTTPReadHeaderTimeout, WriteTimeout: src.HTTPWriteTimeout, + CORS: m.migrateCORS(src.CORS), }, RateLimiting: m.migrateRateLimiting(src.RateLimiters), } diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 58690fa..fe76182 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -30,6 +30,14 @@ import ( // read_timeout: // read_header_timeout: // write_timeout: +// cors: +// allowed_origins: +// allowed_methods: +// allowed_headers: +// exposed_headers: +// allow_credentials: +// max_age: +// options_passthrough: // auth: // options: // oidc: @@ -46,8 +54,7 @@ import ( // jwt: // secret_key: // envs: -// : -// options: # same as root options +// : # same as root options // rate_limiting: // options: // : @@ -63,15 +70,14 @@ import ( // store_max_keys: // per_handler: // envs: -// : -// options: # same as root options +// : # same as root options // clients: // seq_db: // - id: // timeout: // avg_doc_size: // addrs: -// request_retries: +// request_retries_backoff: // backoff: // initial_interval: // max_interval @@ -402,6 +408,7 @@ type Debug struct { ReadHeaderTimeout time.Duration `yaml:"read_header_timeout"` ReadTimeout time.Duration `yaml:"read_timeout"` WriteTimeout time.Duration `yaml:"write_timeout"` + CORS *CORS `yaml:"cors"` } type JWT struct { @@ -451,14 +458,14 @@ type GRPCParams struct { } type SeqDBClient struct { - ID string `yaml:"id"` - Timeout time.Duration `yaml:"timeout"` - AvgDocSize int `yaml:"avg_doc_size"` - Addrs []string `yaml:"addrs"` - RequestRetries int `yaml:"request_retries"` - RetriesBackoff *Backoff `yaml:"backoff"` - ClientMode string `yaml:"client_mode"` - GRPCParams *GRPCParams `yaml:"grpc_params"` + ID string `yaml:"id"` + Timeout time.Duration `yaml:"timeout"` + AvgDocSize int `yaml:"avg_doc_size"` + Addrs []string `yaml:"addrs"` + RequestRetries int `yaml:"request_retries"` + RequestRetriesBackoff *Backoff `yaml:"request_retries_backoff"` + ClientMode string `yaml:"client_mode"` + GRPCParams *GRPCParams `yaml:"grpc_params"` } type Backoff struct { diff --git a/internal/app/config/v2/validate.go b/internal/app/config/v2/validate.go index 77a1dcb..f56ff0a 100644 --- a/internal/app/config/v2/validate.go +++ b/internal/app/config/v2/validate.go @@ -213,7 +213,7 @@ func validateServer(server *Server) error { return fmt.Errorf("server: at least one of http or grpc must be set") } - if server.HTTP.CORS != nil { + if server.HTTP != nil && server.HTTP.CORS != nil { if len(server.HTTP.CORS.AllowedOrigins) == 0 { server.HTTP.CORS.AllowedOrigins = []string{defaultCORSAllowedOrigins} } diff --git a/internal/app/mw/auth.go b/internal/app/mw/auth.go index 0e4ed32..bb16046 100644 --- a/internal/app/mw/auth.go +++ b/internal/app/mw/auth.go @@ -9,7 +9,6 @@ import ( "github.com/ozontech/seq-ui/internal/app/auth" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" ) @@ -44,18 +43,18 @@ func (providers AuthProvidersByEnv) get(ctx context.Context) AuthProviders { return providers[""] } -func NewAuthProvider(ctx context.Context, options config.AuthOptions, caches map[string]cache.Cache) (AuthProviders, error) { +func NewAuthProvider(ctx context.Context, options config.AuthOptions) (AuthProviders, error) { authPrvds := AuthProviders{} if options.JWT != nil { logger.Info("initializing jwt provider") - authPrvds.JwtProvider = auth.NewJWTProvider(options.JWT.SecretKey) + authPrvds.JwtProvider = auth.NewJWTProvider(options.JWT) } if options.OIDC != nil { logger.Info("initializing oidc provider") var err error - authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, options.OIDC, caches[options.OIDC.CacheID]) + authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, options.OIDC) if err != nil { return authPrvds, fmt.Errorf("failed to init oidc provider: %w", err) } diff --git a/internal/app/mw/grpc_mw.go b/internal/app/mw/grpc_mw.go index 55b6b4d..5d0e52d 100644 --- a/internal/app/mw/grpc_mw.go +++ b/internal/app/mw/grpc_mw.go @@ -139,7 +139,7 @@ var noAuthGRPCMethods = map[string]map[string]struct{}{ var errUnauth = status.Error(codes.Unauthenticated, types.ErrUnauthenticated.Error()) -func GRPCAuthInterceptor(providers *AuthProvidersByEnv) grpc.UnaryServerInterceptor { +func GRPCAuthInterceptor(providers AuthProvidersByEnv) grpc.UnaryServerInterceptor { return func( ctx context.Context, req any, info *grpc.UnaryServerInfo, h grpc.UnaryHandler, diff --git a/internal/app/mw/http_mw.go b/internal/app/mw/http_mw.go index daab4c3..4d65569 100644 --- a/internal/app/mw/http_mw.go +++ b/internal/app/mw/http_mw.go @@ -283,11 +283,12 @@ func HTTPEnvInterceptor(resolver EnvResolver) func(next http.Handler) http.Handl fn := func(w http.ResponseWriter, r *http.Request) { uriApi, _, uriMethod, err := parseURI(r.RequestURI) if err != nil { - next.ServeHTTP(w, r) + logger.Error("failed to parse URI", zap.Error(err)) + http.NotFound(w, r) return } - // async ssearch routes are part of seqapi, but have own envs + // async search routes are part of seqapi, but have own envs if uriApi == "seqapi" && uriMethod == "async_search" { uriApi = "async_search" } diff --git a/internal/app/mw/ratelimiter.go b/internal/app/mw/ratelimiter.go index adaa6c5..e2b67b5 100644 --- a/internal/app/mw/ratelimiter.go +++ b/internal/app/mw/ratelimiter.go @@ -22,6 +22,7 @@ var grpcServices = map[string]struct{}{ "DashboardsService": {}, "MassExportService": {}, "ErrorGroupsService": {}, + "AdminService": {}, } type rateLimiter interface { @@ -37,8 +38,8 @@ type RateLimiter struct { type RateLimitersByEnv map[string]map[string]map[string]RateLimiter func (ratelimiters RateLimitersByEnv) get(ctx context.Context) map[string]map[string]RateLimiter { - if ratelimiter, ok := ratelimiters[types.GetEnvKey(ctx)]; ok { - return ratelimiter + if rl, ok := ratelimiters[types.GetEnvKey(ctx)]; ok { + return rl } return ratelimiters[""] } @@ -55,8 +56,8 @@ func NewRateLimiters(apiToRateLimiters config.ApiToRateLimiters) (map[string]map } result[apiName][RateLimiterDefaultUser] = defaultLimiter - for user, rateLimiter := range rateLimiters.SpecialUsers { - limiter, err := newRateLimiter(apiName, rateLimiter) + for user, userRateLimiter := range rateLimiters.SpecialUsers { + limiter, err := newRateLimiter(apiName, userRateLimiter) if err != nil { return nil, fmt.Errorf("init user %q rate limiter: %w", user, err) } diff --git a/internal/app/server/init.go b/internal/app/server/init.go index 781eb88..f668e01 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -28,12 +28,10 @@ func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { if err := s.prepareRateLimiters(); err != nil { return err } - if s.config.GRPC != nil { - s.prepareGRPCServer(registrar) - } - if s.config.GRPC != nil { - s.prepareHTTPServer(ctx, registrar) - } + + s.prepareGRPCServer(registrar) + s.prepareHTTPServer(ctx, registrar) + if err := s.prepareDebugServer(ctx); err != nil { return err } @@ -48,14 +46,14 @@ func (s *Server) prepareAuth(ctx context.Context) error { s.authPrvds = make(mw.AuthProvidersByEnv, len(s.config.Auth.Envs)+1) - provider, err := mw.NewAuthProvider(ctx, s.config.Auth.Options, s.caches) + provider, err := mw.NewAuthProvider(ctx, s.config.Auth.Options) if err != nil { return fmt.Errorf("init auth: %w", err) } s.authPrvds[""] = provider for envName, envConfig := range s.config.Auth.Envs { - provider, err := mw.NewAuthProvider(ctx, envConfig, s.caches) + provider, err := mw.NewAuthProvider(ctx, envConfig) if err != nil { return fmt.Errorf("init auth for env %q: %w", envName, err) } @@ -65,15 +63,19 @@ func (s *Server) prepareAuth(ctx context.Context) error { } // setupCORS applies CORS policies set in config to the provided mux. -func (s *Server) setupCORS(mux *chi.Mux) { +func (s *Server) setupCORS(mux *chi.Mux, cfg *config.CORS) { + if cfg == nil { + return + } + mux.Use(cors.Handler(cors.Options{ - AllowedOrigins: s.config.HTTP.CORS.AllowedOrigins, - AllowedMethods: s.config.HTTP.CORS.AllowedMethods, - AllowedHeaders: s.config.HTTP.CORS.AllowedHeaders, - ExposedHeaders: s.config.HTTP.CORS.ExposedHeaders, - AllowCredentials: s.config.HTTP.CORS.AllowCredentials, - MaxAge: s.config.HTTP.CORS.MaxAge, - OptionsPassthrough: s.config.HTTP.CORS.OptionsPassthrough, + AllowedOrigins: cfg.AllowedOrigins, + AllowedMethods: cfg.AllowedMethods, + AllowedHeaders: cfg.AllowedHeaders, + ExposedHeaders: cfg.ExposedHeaders, + AllowCredentials: cfg.AllowCredentials, + MaxAge: cfg.MaxAge, + OptionsPassthrough: cfg.OptionsPassthrough, })) } @@ -104,13 +106,17 @@ func (s *Server) prepareRateLimiters() error { // prepareGRPCServer creates gRPC server and binds it to the Server instance. func (s *Server) prepareGRPCServer(registrar *api.Registrar) { + if s.config.GRPC != nil { + return + } + interceptors := []grpc.UnaryServerInterceptor{ mw.GRPCRecoverInterceptor(), mw.GRPCProcessHeadersInterceptor(), mw.GRPCEnvInterceptor(s.envResolver), } if len(s.authPrvds) > 0 { - interceptors = append(interceptors, mw.GRPCAuthInterceptor(&s.authPrvds)) + interceptors = append(interceptors, mw.GRPCAuthInterceptor(s.authPrvds)) } interceptors = append(interceptors, mw.GRPCMetricInterceptor(), @@ -135,10 +141,13 @@ func (s *Server) prepareGRPCServer(registrar *api.Registrar) { // prepareHTTPServer prepares HTTP server with applied CORS policies and added interceptors. func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar) { - mux := chi.NewMux() - if s.config.HTTP.CORS != nil { - s.setupCORS(mux) + if s.config.HTTP != nil { + return } + + mux := chi.NewMux() + s.setupCORS(mux, s.config.HTTP.CORS) + interceptors := chi.Middlewares{ mw.HTTPNotFoundInterceptor(), mw.HTTPRecoverInterceptor(), @@ -166,9 +175,7 @@ func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar // prepareHTTPMux prepares debug HTTP server mux with applied CORS policies and added interceptors. func (s *Server) prepareDebugServer(ctx context.Context) error { mux := chi.NewMux() - if s.config.HTTP != nil && s.config.HTTP.CORS != nil { - s.setupCORS(mux) - } + s.setupCORS(mux, s.config.Debug.CORS) interceptors := chi.Middlewares{ mw.HTTPRecoverInterceptor(), } diff --git a/internal/app/server/server.go b/internal/app/server/server.go index 37f09be..19ab581 100644 --- a/internal/app/server/server.go +++ b/internal/app/server/server.go @@ -10,7 +10,6 @@ import ( "github.com/ozontech/seq-ui/internal/api" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/mw" - "github.com/ozontech/seq-ui/internal/pkg/cache" ) // Server contains application dependencies. @@ -19,7 +18,6 @@ type Server struct { debugServer *http.Server grpcServer *grpc.Server httpServer *http.Server - caches map[string]cache.Cache envResolver mw.EnvResolver authPrvds mw.AuthProvidersByEnv @@ -27,10 +25,9 @@ type Server struct { } // New returns a new Server. -func New(ctx context.Context, cfg *config.Config, registrar *api.Registrar, caches map[string]cache.Cache) (*Server, error) { +func New(ctx context.Context, cfg *config.Config, registrar *api.Registrar) (*Server, error) { s := &Server{ config: &cfg.Server, - caches: caches, envResolver: mw.NewEnvResolver(&cfg.Handlers), } diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index 87da6d6..49f7ac8 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -4,64 +4,60 @@ import ( "context" "fmt" - "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config/v2" - "github.com/ozontech/seq-ui/logger" ) -func FromConfig(ctx context.Context, cfg *config.Cache) (map[string]Cache, error) { - caches := make(map[string]Cache, len(cfg.Redis)+len(cfg.Inmemory)) +var caches map[string]Cache + +func Init(ctx context.Context, cfg *config.Cache) error { + caches = make(map[string]Cache, len(cfg.Redis)+len(cfg.Inmemory)) + withInmemIDs := make(map[string]struct{}) - for i := range cfg.Redis { - redis := &cfg.Redis[i] + for _, redis := range cfg.Redis { var inmem *config.Inmemory if redis.WithInmemID != "" { inmem = cfg.InmemByID(redis.WithInmemID) + withInmemIDs[redis.WithInmemID] = struct{}{} } - cache, err := New(ctx, inmem, redis) + c, err := newRedisOrInmemWithRedis(ctx, &redis, inmem) if err != nil { - return nil, fmt.Errorf("new cache %q: %w", redis.ID, err) + return fmt.Errorf("init redis cache %q: %w", redis.ID, err) } - - caches[redis.ID] = cache + caches[redis.ID] = c } - for i := range cfg.Inmemory { - inmem := &cfg.Inmemory[i] - if hasWithInmem(inmem.ID, cfg.Redis) { + for _, inmem := range cfg.Inmemory { + if _, ok := withInmemIDs[inmem.ID]; ok { continue } - cache, err := newInmemoryCache(*inmem) + c, err := newInmemoryCache(inmem) if err != nil { - return nil, fmt.Errorf("new inmemory cache %q: %w", inmem.ID, err) + return fmt.Errorf("init inmemory cache %q: %w", inmem.ID, err) } - - caches[inmem.ID] = cache + caches[inmem.ID] = c } - return caches, nil + return nil } -func New(ctx context.Context, inmemCfg *config.Inmemory, redisCfg *config.Redis) (Cache, error) { - redis, redisErr := newRedisCache(ctx, redisCfg) +func ByID(id string) Cache { + return caches[id] +} + +func newRedisOrInmemWithRedis(ctx context.Context, redisCfg *config.Redis, inmemCfg *config.Inmemory) (Cache, error) { + redis, err := newRedisCache(ctx, redisCfg) + if err != nil { + return nil, err + } if inmemCfg == nil { - if redisErr != nil { - return nil, fmt.Errorf("init redis cache: %w", redisErr) - } return redis, nil } inmem, err := newInmemoryCache(*inmemCfg) if err != nil { - return nil, fmt.Errorf("init inmemory cache: %w", err) - } - - if redisErr != nil { - logger.Warn("failed to init redis cache; inmemory cache will be used instead", zap.Error(redisErr)) - return inmem, redisErr + return nil, fmt.Errorf("init inmemory cache %q: %w", inmemCfg.ID, err) } return &inmemWithRedis{ @@ -69,13 +65,3 @@ func New(ctx context.Context, inmemCfg *config.Inmemory, redisCfg *config.Redis) redis: redis, }, nil } - -func hasWithInmem(inmemID string, redisCfgs []config.Redis) bool { - for i := range redisCfgs { - if redisCfgs[i].WithInmemID == inmemID { - return true - } - } - - return false -} diff --git a/internal/pkg/cache/redis.go b/internal/pkg/cache/redis.go index 6eff755..b82b86b 100644 --- a/internal/pkg/cache/redis.go +++ b/internal/pkg/cache/redis.go @@ -21,11 +21,9 @@ type redisCache struct { func newRedisCache(ctx context.Context, cfg *config.Redis) (*redisCache, error) { client, err := redisclient.New(ctx, cfg) if err != nil { - metric.ServerCacheRedisMisses.Inc() return nil, fmt.Errorf("create redis client: %w", err) } - metric.ServerCacheRedisHits.Inc() return &redisCache{client: client, prefix: cfg.KeyPrefix}, nil } @@ -40,10 +38,12 @@ func (c *redisCache) SetWithTTL(ctx context.Context, key string, value string, t func (c *redisCache) Get(ctx context.Context, key string) (string, error) { value, err := c.client.Get(ctx, c.withPrefix(key)).Result() if err == nil { + metric.ServerCacheRedisMisses.Inc() return value, nil } if errors.Is(err, redis.Nil) { + metric.ServerCacheRedisHits.Inc() return "", ErrNotFound } From e5d6c34a2195dc7d386745163657cb62d36a845b Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 14 Oct 2026 14:45:02 +0300 Subject: [PATCH 20/29] update auth/config/midrate/cache --- internal/app/auth/oidc.go | 5 +- internal/app/config/migrate/v1_v2.go | 247 +++++++++--------- internal/app/config/v2/config.go | 6 +- internal/app/config/v2/validate.go | 10 +- internal/app/server/init.go | 4 +- internal/pkg/cache/ctor.go | 57 ++-- internal/pkg/cache/inmemory.go | 2 +- internal/pkg/cache/redis.go | 16 +- ...nmem_with_redis.go => redis_with_inmem.go} | 16 +- 9 files changed, 181 insertions(+), 182 deletions(-) rename internal/pkg/cache/{inmem_with_redis.go => redis_with_inmem.go} (76%) diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 3589f39..783b19f 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -45,12 +45,13 @@ type oidcProvider struct { func NewOIDCProvider(ctx context.Context, cfg *config.OIDC) (OIDCProvider, error) { var ( err error - oidcCtx context.Context oidcCache cache.Cache + oidcCtx context.Context ) if cfg.CacheID != "" { - if oidcCache = cache.ByID(cfg.CacheID); oidcCache == nil { + var found bool + if oidcCache, found = cache.ByID(cfg.CacheID); !found { return nil, fmt.Errorf("oidc cache %q not found", cfg.CacheID) } } diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index b54b95e..df613a7 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -1,6 +1,8 @@ package migrate import ( + "time" + v1 "github.com/ozontech/seq-ui/internal/app/config/v1" v2 "github.com/ozontech/seq-ui/internal/app/config/v2" ) @@ -18,14 +20,14 @@ func (m *v1tov2) migrate() v2.Config { dst := v2.Config{Version: 2} dst.Server = m.migrateServer(m.src.Server) - // var v1Cache v1.Cache - // if m.src.Server != nil { - // v1Cache = m.src.Server.Cache - // } - // - // dst.Cache = m.migrateCache(v1Cache) - // dst.Clients = m.migrateClients(m.src) - // dst.DB = m.migrateDB(m.src.Server) + var v1Cache v1.Cache + if m.src.Server != nil { + v1Cache = m.src.Server.Cache + } + + dst.Cache = m.migrateCache(v1Cache) + dst.Clients = m.migrateClients(m.src) + dst.DB = m.migrateDB(m.src.Server) // dst.Handlers = m.migrateHandlers(m.src.Handlers, &dst, v1Cache) return dst @@ -166,114 +168,123 @@ func (m *v1tov2) migrateGRPCParams(src *v1.GRPCKeepaliveParams) *v2.GRPCParams { } } -// func (m *v1tov2) migrateClients(src v1.Config) v2.Clients { -// dst := v2.Clients{} -// -// if src.Clients != nil { -// if len(src.Clients.SeqDB) > 0 { -// dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) -// for i := range src.Clients.SeqDB { -// dst.SeqDB = append(dst.SeqDB, m.migrateSeqDBClient(&src.Clients.SeqDB[i])) -// } -// } else { -// dst.SeqDB = []v2.SeqDBClient{{ -// ID: v2.DefaultSeqDBClientID, -// Timeout: src.Clients.SeqDBTimeout, -// AvgDocSize: src.Clients.SeqDBAvgDocSize, -// Addrs: src.Clients.SeqDBAddrs, -// RequestRetries: src.Clients.RequestRetries, -// InitialRetryBackoff: src.Clients.InitialRetryBackoff, -// MaxRetryBackoff: src.Clients.MaxRetryBackoff, -// ClientMode: src.Clients.ProxyClientMode, -// GRPCParams: m.migrateGRPCParams(src.Clients.GRPCKeepaliveParams), -// }} -// } -// } -// -// if src.Server != nil && src.Server.CH != nil { -// ch := src.Server.CH -// dst.ClickHouse = []v2.CHClient{{ -// ID: v2.DefaultCHClientID, -// Addrs: ch.Addrs, -// Database: ch.Database, -// Username: ch.Username, -// Password: ch.Password, -// Sharded: ch.Sharded, -// DialTimeout: ch.DialTimeout, -// ReadTimeout: ch.ReadTimeout, -// }} -// } -// -// return dst -// } -// -// func (m *v1tov2) migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { -// return v2.SeqDBClient{ -// ID: src.ID, -// Timeout: src.Timeout, -// AvgDocSize: src.AvgDocSize, -// Addrs: src.Addrs, -// RequestRetries: src.RequestRetries, -// InitialRetryBackoff: src.InitialRetryBackoff, -// MaxRetryBackoff: src.MaxRetryBackoff, -// ClientMode: src.ClientMode, -// GRPCParams: m.migrateGRPCParams(src.GRPCKeepaliveParams), -// } -// } -// -// func (m *v1tov2) migrateDB(src *v1.Server) *v2.DB { -// if src == nil || src.DB == nil { -// return nil -// } -// -// return &v2.DB{ -// Name: src.DB.Name, -// Host: src.DB.Host, -// Port: src.DB.Port, -// Pass: src.DB.Pass, -// User: src.DB.User, -// RequestTimeout: src.DB.RequestTimeout, -// ConnectionPoolCapacity: src.DB.ConnectionPoolCapacity, -// UsePreparedStatements: src.DB.UsePreparedStatements, -// } -// } -// -// func (m *v1tov2) migrateCache(src v1.Cache) v2.Cache { -// // in config v1 inmemory cache is always created -// cache := v2.Cache{ -// Inmemory: []v2.InmemoryCache{{ -// ID: v2.DefaultInmemCacheID, -// NumCounters: src.Inmemory.NumCounters, -// MaxCost: src.Inmemory.MaxCost, -// BufferItems: src.Inmemory.BufferItems, -// }}, -// } -// -// if src.Redis != nil { -// cache.Redis = append(cache.Redis, -// m.migrateRedis(src.Redis, v2.DefaultRedisID, v2.DefaultInmemCacheID), -// m.migrateRedis(src.Redis, v2.DefaultRedis2ID, ""), -// ) -// } -// -// return cache -// } -// -// func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { -// return v2.Redis{ -// ID: id, -// WithInmemID: withInmemID, -// Addr: src.Addr, -// Username: src.Username, -// Password: src.Password, -// Timeout: src.Timeout, -// MaxRetries: src.MaxRetries, -// MinRetryBackoff: src.MinRetryBackoff, -// MaxRetryBackoff: src.MaxRetryBackoff, -// KeyPrefix: src.KeyPrefix, -// } -// } -// +func (m *v1tov2) migrateClients(src v1.Config) v2.Clients { + dst := v2.Clients{} + + if src.Clients != nil { + if len(src.Clients.SeqDB) > 0 { + dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) + for i := range src.Clients.SeqDB { + dst.SeqDB = append(dst.SeqDB, m.migrateSeqDBClient(&src.Clients.SeqDB[i])) + } + } else { + dst.SeqDB = []v2.SeqDBClient{{ + ID: v2.DefaultSeqDBClientID, + Timeout: src.Clients.SeqDBTimeout, + AvgDocSize: src.Clients.SeqDBAvgDocSize, + Addrs: src.Clients.SeqDBAddrs, + RequestRetries: src.Clients.RequestRetries, + RequestRetriesBackoff: m.migrateBackoff(src.Clients.InitialRetryBackoff, src.Clients.MaxRetryBackoff), + ClientMode: src.Clients.ProxyClientMode, + GRPCParams: m.migrateGRPCParams(src.Clients.GRPCKeepaliveParams), + }} + } + } + + if src.Server != nil && src.Server.CH != nil { + ch := src.Server.CH + dst.ClickHouse = []v2.CHClient{{ + ID: v2.DefaultCHClientID, + Addrs: ch.Addrs, + Database: ch.Database, + Username: ch.Username, + Password: ch.Password, + Sharded: ch.Sharded, + DialTimeout: ch.DialTimeout, + ReadTimeout: ch.ReadTimeout, + }} + } + + return dst +} + +func (m *v1tov2) migrateBackoff(initialInterval, maxInterval time.Duration) *v2.Backoff { + if initialInterval == 0 && maxInterval == 0 { + return nil + } + + return &v2.Backoff{ + InitialInterval: initialInterval, + MaxInterval: maxInterval, + } +} + +func (m *v1tov2) migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { + return v2.SeqDBClient{ + ID: src.ID, + Timeout: src.Timeout, + AvgDocSize: src.AvgDocSize, + Addrs: src.Addrs, + RequestRetries: src.RequestRetries, + RequestRetriesBackoff: m.migrateBackoff(src.InitialRetryBackoff, src.MaxRetryBackoff), + ClientMode: src.ClientMode, + GRPCParams: m.migrateGRPCParams(src.GRPCKeepaliveParams), + } +} + +func (m *v1tov2) migrateDB(src *v1.Server) *v2.DB { + if src == nil || src.DB == nil { + return nil + } + + return &v2.DB{ + Name: src.DB.Name, + Host: src.DB.Host, + Port: src.DB.Port, + User: src.DB.User, + Pass: src.DB.Pass, + RequestTimeout: src.DB.RequestTimeout, + ConnectionPoolCapacity: src.DB.ConnectionPoolCapacity, + UsePreparedStatements: src.DB.UsePreparedStatements, + } +} + +func (m *v1tov2) migrateCache(src v1.Cache) v2.Cache { + // in config v1 inmemory cache is always created + cache := v2.Cache{ + Inmemory: []v2.Inmemory{{ + ID: v2.DefaultInmemID, + NumCounters: src.Inmemory.NumCounters, + MaxCost: src.Inmemory.MaxCost, + BufferItems: src.Inmemory.BufferItems, + }}, + } + + if src.Redis != nil { + cache.Redis = append(cache.Redis, + m.migrateRedis(src.Redis, v2.DefaultRedisWithInmemID, v2.DefaultInmemID), + m.migrateRedis(src.Redis, v2.DefaultRedisWithoutInmemID, ""), + ) + } + + return cache +} + +func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { + return v2.Redis{ + ID: id, + WithInmemID: withInmemID, + Addr: src.Addr, + Username: src.Username, + Password: src.Password, + Timeout: src.Timeout, + MaxRetries: src.MaxRetries, + MinRetryBackoff: src.MinRetryBackoff, + MaxRetryBackoff: src.MaxRetryBackoff, + KeyPrefix: src.KeyPrefix, + } +} + // func (m *v1tov2) migrateHandlers(src *v1.Handlers, cfg *v2.Config, v1Cache v1.Cache) v2.Handlers { // if src == nil { // return v2.Handlers{} @@ -566,9 +577,9 @@ func (m *v1tov2) migrateGRPCParams(src *v1.GRPCKeepaliveParams) *v2.GRPCParams { // redisOrInmemID - redis or inmemory (v1 logslifespan) func (m *v1tov2) defaultCacheIDs(src v1.Cache) (inmemWithRedisID, redisOrInmemID string) { if src.Redis != nil { - return v2.DefaultRedisID, v2.DefaultRedis2ID + return v2.DefaultRedisWithInmemID, v2.DefaultRedisWithoutInmemID } - return v2.DefaultInmemCacheID, v2.DefaultInmemCacheID + return v2.DefaultInmemID, v2.DefaultInmemID } // diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index fe76182..10edcff 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -560,9 +560,9 @@ type SeqAPILimits struct { } type SeqAPICaches struct { - Events HandlerCache `yaml:"events"` - LogsLifespan HandlerCache `yaml:"logs_lifespan"` - Fields HandlerCache `yaml:"fields"` + Events *HandlerCache `yaml:"events"` + LogsLifespan *HandlerCache `yaml:"logs_lifespan"` + Fields *HandlerCache `yaml:"fields"` } type HandlerCache struct { diff --git a/internal/app/config/v2/validate.go b/internal/app/config/v2/validate.go index f56ff0a..023ff2a 100644 --- a/internal/app/config/v2/validate.go +++ b/internal/app/config/v2/validate.go @@ -8,11 +8,11 @@ import ( const ( MaxHTTPHeaderBytes = 1 << 12 // 4 KiB - DefaultSeqDBClientID = "default_seq_db" - DefaultCHClientID = "default_ch" - DefaultInmemCacheID = "default_inmem_cache" - DefaultRedisID = "default_redis" - DefaultRedis2ID = "default_redis_2" + DefaultSeqDBClientID = "default_seq_db" + DefaultCHClientID = "default_ch" + DefaultInmemID = "default_inmem" + DefaultRedisWithInmemID = "default_redis_with_inmem" + DefaultRedisWithoutInmemID = "default_redis_without_inmem" SeqDBClientModeGRPC = "grpc" diff --git a/internal/app/server/init.go b/internal/app/server/init.go index f668e01..b21a3e4 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -106,7 +106,7 @@ func (s *Server) prepareRateLimiters() error { // prepareGRPCServer creates gRPC server and binds it to the Server instance. func (s *Server) prepareGRPCServer(registrar *api.Registrar) { - if s.config.GRPC != nil { + if s.config.GRPC == nil { return } @@ -141,7 +141,7 @@ func (s *Server) prepareGRPCServer(registrar *api.Registrar) { // prepareHTTPServer prepares HTTP server with applied CORS policies and added interceptors. func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar) { - if s.config.HTTP != nil { + if s.config.HTTP == nil { return } diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index 49f7ac8..0601dfc 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -11,57 +11,36 @@ var caches map[string]Cache func Init(ctx context.Context, cfg *config.Cache) error { caches = make(map[string]Cache, len(cfg.Redis)+len(cfg.Inmemory)) - withInmemIDs := make(map[string]struct{}) - for _, redis := range cfg.Redis { - var inmem *config.Inmemory - if redis.WithInmemID != "" { - inmem = cfg.InmemByID(redis.WithInmemID) - withInmemIDs[redis.WithInmemID] = struct{}{} - } - - c, err := newRedisOrInmemWithRedis(ctx, &redis, inmem) + for i := range cfg.Inmemory { + inmemCfg := &cfg.Inmemory[i] + inmem, err := newInmemoryCache(inmemCfg) if err != nil { - return fmt.Errorf("init redis cache %q: %w", redis.ID, err) + return fmt.Errorf("init inmemory cache %q: %w", inmemCfg.ID, err) } - caches[redis.ID] = c + caches[inmemCfg.ID] = inmem } - for _, inmem := range cfg.Inmemory { - if _, ok := withInmemIDs[inmem.ID]; ok { - continue + for i := range cfg.Redis { + redisCfg := &cfg.Redis[i] + + var inmem Cache + if redisCfg.WithInmemID != "" { + inmem = caches[redisCfg.WithInmemID] } - c, err := newInmemoryCache(inmem) + redis, err := newRedisCache(ctx, redisCfg, inmem) if err != nil { - return fmt.Errorf("init inmemory cache %q: %w", inmem.ID, err) + return fmt.Errorf("init redis cache %q: %w", redisCfg.ID, err) } - caches[inmem.ID] = c + caches[redisCfg.ID] = redis } return nil } -func ByID(id string) Cache { - return caches[id] -} - -func newRedisOrInmemWithRedis(ctx context.Context, redisCfg *config.Redis, inmemCfg *config.Inmemory) (Cache, error) { - redis, err := newRedisCache(ctx, redisCfg) - if err != nil { - return nil, err - } - if inmemCfg == nil { - return redis, nil - } - - inmem, err := newInmemoryCache(*inmemCfg) - if err != nil { - return nil, fmt.Errorf("init inmemory cache %q: %w", inmemCfg.ID, err) - } - - return &inmemWithRedis{ - inmem: inmem, - redis: redis, - }, nil +// ByID returns cache by id and true, false if cache is not found. +func ByID(id string) (Cache, bool) { + cache, ok := caches[id] + return cache, ok } diff --git a/internal/pkg/cache/inmemory.go b/internal/pkg/cache/inmemory.go index 8250f02..d58a9b0 100644 --- a/internal/pkg/cache/inmemory.go +++ b/internal/pkg/cache/inmemory.go @@ -17,7 +17,7 @@ type inmemoryCache struct { cache *ristretto.Cache[string, string] } -func newInmemoryCache(cfg config.Inmemory) (*inmemoryCache, error) { +func newInmemoryCache(cfg *config.Inmemory) (Cache, error) { c, err := ristretto.NewCache(&ristretto.Config[string, string]{ NumCounters: cfg.NumCounters, MaxCost: cfg.MaxCost, diff --git a/internal/pkg/cache/redis.go b/internal/pkg/cache/redis.go index b82b86b..2676592 100644 --- a/internal/pkg/cache/redis.go +++ b/internal/pkg/cache/redis.go @@ -18,13 +18,21 @@ type redisCache struct { prefix string } -func newRedisCache(ctx context.Context, cfg *config.Redis) (*redisCache, error) { +func newRedisCache(ctx context.Context, cfg *config.Redis, inmem Cache) (Cache, error) { client, err := redisclient.New(ctx, cfg) if err != nil { return nil, fmt.Errorf("create redis client: %w", err) } - return &redisCache{client: client, prefix: cfg.KeyPrefix}, nil + rc := &redisCache{client: client, prefix: cfg.KeyPrefix} + if inmem == nil { + return rc, nil + } + + return &redisWithInmem{ + redis: rc, + inmem: inmem, + }, nil } func (c *redisCache) Set(ctx context.Context, key string, value string) error { @@ -38,12 +46,12 @@ func (c *redisCache) SetWithTTL(ctx context.Context, key string, value string, t func (c *redisCache) Get(ctx context.Context, key string) (string, error) { value, err := c.client.Get(ctx, c.withPrefix(key)).Result() if err == nil { - metric.ServerCacheRedisMisses.Inc() + metric.ServerCacheRedisHits.Inc() return value, nil } if errors.Is(err, redis.Nil) { - metric.ServerCacheRedisHits.Inc() + metric.ServerCacheRedisMisses.Inc() return "", ErrNotFound } diff --git a/internal/pkg/cache/inmem_with_redis.go b/internal/pkg/cache/redis_with_inmem.go similarity index 76% rename from internal/pkg/cache/inmem_with_redis.go rename to internal/pkg/cache/redis_with_inmem.go index 304a984..f2422ac 100644 --- a/internal/pkg/cache/inmem_with_redis.go +++ b/internal/pkg/cache/redis_with_inmem.go @@ -10,16 +10,16 @@ import ( "github.com/ozontech/seq-ui/logger" ) -type inmemWithRedis struct { - inmem *inmemoryCache - redis *redisCache +type redisWithInmem struct { + redis Cache + inmem Cache } -func (c *inmemWithRedis) Set(ctx context.Context, key string, value string) error { +func (c *redisWithInmem) Set(ctx context.Context, key string, value string) error { return c.SetWithTTL(ctx, key, value, 0) } -func (c *inmemWithRedis) SetWithTTL(ctx context.Context, key string, value string, ttl time.Duration) error { +func (c *redisWithInmem) SetWithTTL(ctx context.Context, key string, value string, ttl time.Duration) error { if err := c.redis.SetWithTTL(ctx, key, value, ttl); err != nil { logger.Error("failed to set value in redis sub cache", zap.String("key", key), zap.Error(err)) } @@ -27,7 +27,7 @@ func (c *inmemWithRedis) SetWithTTL(ctx context.Context, key string, value strin return c.inmem.SetWithTTL(ctx, key, value, ttl) } -func (c *inmemWithRedis) Get(ctx context.Context, key string) (string, error) { +func (c *redisWithInmem) Get(ctx context.Context, key string) (string, error) { val, err := c.inmem.Get(ctx, key) if err == nil { return val, nil @@ -49,7 +49,7 @@ func (c *inmemWithRedis) Get(ctx context.Context, key string) (string, error) { return "", err } -func (c *inmemWithRedis) GetTTL(ctx context.Context, key string) (time.Duration, error) { +func (c *redisWithInmem) GetTTL(ctx context.Context, key string) (time.Duration, error) { val, err := c.inmem.GetTTL(ctx, key) if err == nil { return val, nil @@ -67,7 +67,7 @@ func (c *inmemWithRedis) GetTTL(ctx context.Context, key string) (time.Duration, return 0, err } -func (c *inmemWithRedis) Del(ctx context.Context, key string) { +func (c *redisWithInmem) Del(ctx context.Context, key string) { c.redis.Del(ctx, key) c.inmem.Del(ctx, key) } From 44a708b3a33498c754b1f6cde2d6667ddf171fdc Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 14 Oct 2026 23:29:23 +0300 Subject: [PATCH 21/29] update cmd, api/seqapi-async, app/config-types, pkg/repository-service, migration --- internal/api/seqapi/v1/api_error/error.go | 6 +- internal/api/seqapi/v1/grpc/aggregation.go | 8 +- .../api/seqapi/v1/grpc/aggregation_test.go | 20 ++- internal/api/seqapi/v1/grpc/api.go | 152 ++++++++---------- .../api/seqapi/v1/grpc/cancel_async_search.go | 5 +- .../api/seqapi/v1/grpc/delete_async_search.go | 5 +- internal/api/seqapi/v1/grpc/events.go | 32 ++-- internal/api/seqapi/v1/grpc/events_test.go | 24 +-- .../v1/grpc/fetch_async_search_result.go | 5 +- internal/api/seqapi/v1/grpc/fields.go | 75 ++++++--- internal/api/seqapi/v1/grpc/fields_test.go | 90 ++++++----- .../seqapi/v1/grpc/get_async_searches_list.go | 5 +- internal/api/seqapi/v1/grpc/get_envs_test.go | 72 +++++---- internal/api/seqapi/v1/grpc/limits.go | 10 +- internal/api/seqapi/v1/grpc/limits_test.go | 12 +- internal/api/seqapi/v1/grpc/logs_lifespan.go | 32 ++-- .../api/seqapi/v1/grpc/logs_lifespan_test.go | 8 +- internal/api/seqapi/v1/grpc/search.go | 14 +- internal/api/seqapi/v1/grpc/search_test.go | 32 ++-- .../api/seqapi/v1/grpc/start_async_search.go | 5 +- internal/api/seqapi/v1/grpc/test_data.go | 33 ++-- internal/api/seqapi/v1/http/api.go | 73 +++++---- .../api/seqapi/v1/http/cancel_async_search.go | 27 ++-- .../api/seqapi/v1/http/delete_async_search.go | 26 +-- internal/api/seqapi/v1/http/events.go | 38 ++--- internal/api/seqapi/v1/http/events_test.go | 24 +-- internal/api/seqapi/v1/http/export.go | 1 - .../api/seqapi/v1/http/export_async_search.go | 14 +- .../v1/http/fetch_async_search_result.go | 17 +- internal/api/seqapi/v1/http/fields.go | 6 +- internal/api/seqapi/v1/http/fields_test.go | 30 ++-- .../seqapi/v1/http/get_async_searches_list.go | 15 +- internal/api/seqapi/v1/http/limits.go | 4 +- internal/api/seqapi/v1/http/logs_lifespan.go | 37 +++-- .../api/seqapi/v1/http/logs_lifespan_test.go | 4 +- internal/api/seqapi/v1/http/search.go | 6 +- internal/api/seqapi/v1/http/search_test.go | 50 ++++-- .../api/seqapi/v1/http/start_async_search.go | 16 +- internal/api/seqapi/v1/http/test_data.go | 33 ++-- internal/api/seqapi/v1/seqapi.go | 41 ++++- 40 files changed, 659 insertions(+), 448 deletions(-) diff --git a/internal/api/seqapi/v1/api_error/error.go b/internal/api/seqapi/v1/api_error/error.go index c203b8c..915c567 100644 --- a/internal/api/seqapi/v1/api_error/error.go +++ b/internal/api/seqapi/v1/api_error/error.go @@ -12,21 +12,21 @@ var ( ErrOffsetConflict = errors.New(`only one of "offset" and "offset_id" must be provided`) ) -func CheckSearchLimit(limit, maximum int) error { +func CheckSearchLimit(limit, maximum int32) error { if limit == 0 || limit > maximum { return fmt.Errorf("'limit' must be greater than 0 and less than %v", maximum) } return nil } -func CheckSearchOffsetLimit(offset, maximum int) error { +func CheckSearchOffsetLimit(offset, maximum int32) error { if offset > maximum { return ErrQueryTooHeavy } return nil } -func CheckForOffsetConflict(offset int, offsetID string) error { +func CheckForOffsetConflict(offset int32, offsetID string) error { if offset != 0 && offsetID != "" { return ErrOffsetConflict } diff --git a/internal/api/seqapi/v1/grpc/aggregation.go b/internal/api/seqapi/v1/grpc/aggregation.go index 3ae1f21..53c8101 100644 --- a/internal/api/seqapi/v1/grpc/aggregation.go +++ b/internal/api/seqapi/v1/grpc/aggregation.go @@ -60,7 +60,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ return nil, status.Error(codes.InvalidArgument, err.Error()) } - if err := api_error.CheckAggregationsCount(len(req.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(req.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -70,7 +70,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ continue } if err := api_error.CheckAggregationTsInterval(*agg.Interval, fromRaw, toRaw, - params.options.MaxBucketsPerAggregationTs, + params.options.Limits.MaxBucketsPerAggregationTs, ); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -81,7 +81,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ return nil, err } - if a.globalParams.masker != nil { + if params.masker != nil { buf := make([]string, 0) for i, agg := range resp.Aggregations { if agg == nil { @@ -99,7 +99,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ field = aggReq.GroupBy } - buf = a.globalParams.masker.MaskAgg(field, buf) + buf = params.masker.MaskAgg(field, buf) for j, key := range buf { if agg.Buckets[j] != nil { diff --git a/internal/api/seqapi/v1/grpc/aggregation_test.go b/internal/api/seqapi/v1/grpc/aggregation_test.go index bc1261a..f4c1a0e 100644 --- a/internal/api/seqapi/v1/grpc/aggregation_test.go +++ b/internal/api/seqapi/v1/grpc/aggregation_test.go @@ -51,7 +51,9 @@ func TestGetAggregation(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -66,7 +68,9 @@ func TestGetAggregation(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 2, + }, }, }, apiErr: true, @@ -83,7 +87,9 @@ func TestGetAggregation(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 1, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 1, + }, }, }, clientErr: errors.New("client error"), @@ -179,7 +185,9 @@ func TestGetAggregationWithNormalization(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -222,7 +230,9 @@ func TestGetAggregationWithNormalization(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index 642e9e0..30bf509 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -6,8 +6,6 @@ import ( "strconv" "time" - "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" @@ -16,84 +14,94 @@ import ( asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" + "go.uber.org/zap" ) type apiParams struct { - client seqdb.Client - options *config.SeqAPIOptions -} - -type apiGlobalParams struct { - fieldsCache *fieldsCache + client seqdb.Client + options *config.SeqAPIOptions masker *mask.Masker pinnedFields []*seqapi.Field systemFields []*seqapi.Field - eventsCacheTTL time.Duration - logsLifespanCacheTTL time.Duration + eventsCache cache.Cache + logsLifespanCache cache.Cache + fieldsCache *fieldsCache + eventsCacheKeyPrefix string + fieldsCacheKey string + logsLifespanCacheKey string } type API struct { seqapi.UnimplementedSeqAPIServiceServer - config *config.SeqAPI - globalParams apiGlobalParams - paramsByEnv map[string]apiParams - inmemWithRedisCache cache.Cache - redisCache cache.Cache - nowFn func() time.Time - asyncSearches asyncsearches.Service - envsResponse *seqapi.GetEnvsResponse + config *config.SeqAPI + paramsByEnv map[string]apiParams + nowFn func() time.Time + asyncSearches map[string]asyncsearches.Service + envsResponse *seqapi.GetEnvsResponse } -func New( - cfg *config.SeqAPI, - seqDBСlients map[string]seqdb.Client, - inmemWithRedisCache cache.Cache, - redisCache cache.Cache, - fieldsCache cache.Cache, - asyncSearches asyncsearches.Service, -) *API { - globalParams := apiGlobalParams{ - pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), - systemFields: parseFields(cfg.GlobalOptions.SystemFields), - eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, - logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, +func New(cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, asyncSearches map[string]asyncsearches.Service) *API { + paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envConfig := range cfg.Envs { + paramsByEnv[envName] = newAPIParams(envName, seqDBСlients[envConfig.SeqDBID], envConfig.Options) + } + } else { + var client seqdb.Client + for _, c := range seqDBСlients { + client = c + } + paramsByEnv[""] = newAPIParams("", client, &cfg.Options) } - if cfg.GlobalOptions.FieldsCacheTTL > 0 { - globalParams.fieldsCache = newFieldsCache(cfg.GlobalOptions.FieldsCacheTTL) + return &API{ + config: cfg, + paramsByEnv: paramsByEnv, + nowFn: time.Now, + asyncSearches: asyncSearches, + envsResponse: parseEnvs(cfg), } +} - globalMasker, err := mask.New(cfg.GlobalOptions.Masking) +func newAPIParams(env string, client seqdb.Client, options *config.SeqAPIOptions) apiParams { + masker, err := mask.New(options.Masking) if err != nil { - logger.Fatal("failed to init masking", zap.Error(err)) + logger.Fatal("failed to init masking", zap.String("env", env), zap.Error(err)) } - globalParams.masker = globalMasker - paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) - if len(cfg.Envs) > 0 { - for envName, envConfig := range cfg.Envs { - paramsByEnv[envName] = apiParams{ - client: seqDBСlients[envConfig.SeqDBID], - options: envConfig.Options, - } - } - } else { - paramsByEnv[""] = apiParams{ - client: seqDBСlients[cfg.SeqDBID], - options: &cfg.Options, + var eventsCache, logsLifespanCache cache.Cache + if options.Caches.Events != nil { + eventsCache, _ = cache.ByID(options.Caches.Events.ID) + } + if options.Caches.LogsLifespan != nil { + logsLifespanCache, _ = cache.ByID(options.Caches.LogsLifespan.ID) + } + + var fc *fieldsCache + if options.Caches.Fields != nil { + if c, ok := cache.ByID(options.Caches.Fields.ID); ok { + fc = newFieldsCache(c) } } - return &API{ - config: cfg, - globalParams: globalParams, - paramsByEnv: paramsByEnv, - inmemWithRedisCache: inmemWithRedisCache, - redisCache: redisCache, - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + cacheKeyPrefix := "seqapi_" + if env != "" { + cacheKeyPrefix += env + "_" + } + + return apiParams{ + client: client, + options: options, + masker: masker, + pinnedFields: parseFields(options.PinnedFields), + systemFields: parseFields(options.SystemFields), + eventsCache: eventsCache, + logsLifespanCache: logsLifespanCache, + fieldsCache: fc, + eventsCacheKeyPrefix: cacheKeyPrefix + "event_", + fieldsCacheKey: cacheKeyPrefix + "fields", + logsLifespanCacheKey: cacheKeyPrefix + "logs_lifespan", } } @@ -174,26 +182,6 @@ func (a *API) GetParams(env string) (apiParams, error) { return params, nil } -type fieldsCache struct { - ttl time.Duration - - ts time.Time - fields []*seqapi.Field -} - -func newFieldsCache(ttl time.Duration) *fieldsCache { - return &fieldsCache{ttl: ttl} -} - -func (c *fieldsCache) getFields() ([]*seqapi.Field, bool, bool) { - return c.fields, !c.ts.IsZero(), time.Since(c.ts) < c.ttl -} - -func (c *fieldsCache) setFields(fields []*seqapi.Field) { - c.fields = fields - c.ts = time.Now() -} - func checkLimitOffset(limit, offset int32) error { if limit < 0 { return types.NewErrInvalidRequestField("'limit' must be non-negative") @@ -207,10 +195,10 @@ func checkLimitOffset(limit, offset int32) error { func createEnvInfo(envName string, opts *config.SeqAPIOptions) *seqapi.GetEnvsResponse_Env { return &seqapi.GetEnvsResponse_Env{ Env: envName, - MaxSearchLimit: uint32(opts.MaxSearchLimit), - MaxExportLimit: uint32(opts.MaxExportLimit), - MaxParallelExportRequests: uint32(opts.MaxParallelExportRequests), - MaxAggregationsPerRequest: uint32(opts.MaxAggregationsPerRequest), - SeqCliMaxSearchLimit: uint32(opts.SeqCLIMaxSearchLimit), + MaxSearchLimit: uint32(opts.Limits.MaxSearchLimit), + MaxExportLimit: uint32(opts.Limits.MaxExportLimit), + MaxParallelExportRequests: uint32(opts.Limits.MaxParallelExportRequests), + MaxAggregationsPerRequest: uint32(opts.Limits.MaxAggregationsPerRequest), + SeqCliMaxSearchLimit: uint32(opts.Limits.SeqCLIMaxSearchLimit), } } diff --git a/internal/api/seqapi/v1/grpc/cancel_async_search.go b/internal/api/seqapi/v1/grpc/cancel_async_search.go index 2506f36..00f2838 100644 --- a/internal/api/seqapi/v1/grpc/cancel_async_search.go +++ b/internal/api/seqapi/v1/grpc/cancel_async_search.go @@ -15,7 +15,8 @@ import ( ) func (a *API) CancelAsyncSearch(ctx context.Context, req *seqapi.CancelAsyncSearchRequest) (*seqapi.CancelAsyncSearchResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -33,7 +34,7 @@ func (a *API) CancelAsyncSearch(ctx context.Context, req *seqapi.CancelAsyncSear }, ) - resp, err := a.asyncSearches.CancelAsyncSearch(ctx, req) + resp, err := asyncSearches.CancelAsyncSearch(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/delete_async_search.go b/internal/api/seqapi/v1/grpc/delete_async_search.go index 4c573ea..63f2fed 100644 --- a/internal/api/seqapi/v1/grpc/delete_async_search.go +++ b/internal/api/seqapi/v1/grpc/delete_async_search.go @@ -15,7 +15,8 @@ import ( ) func (a *API) DeleteAsyncSearch(ctx context.Context, req *seqapi.DeleteAsyncSearchRequest) (*seqapi.DeleteAsyncSearchResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -33,7 +34,7 @@ func (a *API) DeleteAsyncSearch(ctx context.Context, req *seqapi.DeleteAsyncSear }, ) - resp, err := a.asyncSearches.DeleteAsyncSearch(ctx, req) + resp, err := asyncSearches.DeleteAsyncSearch(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/events.go b/internal/api/seqapi/v1/grpc/events.go index bda573b..a604a95 100644 --- a/internal/api/seqapi/v1/grpc/events.go +++ b/internal/api/seqapi/v1/grpc/events.go @@ -39,29 +39,35 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap return nil, status.Error(codes.InvalidArgument, err.Error()) } - if cached, err := a.inmemWithRedisCache.Get(ctx, req.Id); err == nil { - event := &seqapi.Event{} - if err = proto.Unmarshal([]byte(cached), event); err == nil { - if a.globalParams.masker != nil { - a.globalParams.masker.Mask(event.Data) + cacheKey := params.eventsCacheKeyPrefix + req.Id + if params.eventsCache != nil { + if cached, err := params.eventsCache.Get(ctx, cacheKey); err == nil { + event := &seqapi.Event{} + if err = proto.Unmarshal([]byte(cached), event); err == nil { + if params.masker != nil { + params.masker.Mask(event.Data) + } + return &seqapi.GetEventResponse{Event: event}, nil } - return &seqapi.GetEventResponse{Event: event}, nil + logger.Error("failed to unmarshal cached event proto", zap.String("id", req.Id), zap.Error(err)) } - logger.Error("failed to unmarshal cached event proto", zap.String("id", req.Id), zap.Error(err)) } + resp, err := params.client.GetEvent(ctx, req) if err != nil { return nil, err } - if a.globalParams.masker != nil && resp.Event != nil { - a.globalParams.masker.Mask(resp.Event.Data) + if params.masker != nil && resp.Event != nil { + params.masker.Mask(resp.Event.Data) } - if data, err := proto.Marshal(resp.Event); err == nil { - _ = a.inmemWithRedisCache.SetWithTTL(ctx, req.Id, string(data), a.globalParams.eventsCacheTTL) - } else { - logger.Error("failed to marshal event proto for caching", zap.String("id", req.Id), zap.Error(err)) + if params.eventsCache != nil { + if data, err := proto.Marshal(resp.Event); err == nil { + _ = params.eventsCache.SetWithTTL(ctx, cacheKey, string(data), params.options.Caches.Events.TTL) + } else { + logger.Error("failed to marshal event proto for caching", zap.String("id", req.Id), zap.Error(err)) + } } return resp, nil diff --git a/internal/api/seqapi/v1/grpc/events_test.go b/internal/api/seqapi/v1/grpc/events_test.go index 736b6c8..cf84867 100644 --- a/internal/api/seqapi/v1/grpc/events_test.go +++ b/internal/api/seqapi/v1/grpc/events_test.go @@ -53,7 +53,7 @@ func TestGetEvent(t *testing.T) { Event: event1, }, cacheArgs: test.CacheMockArgs{ - Key: id1, + Key: "seqapi_event_" + id1, Value: string(event1json), Err: errSomethingWrong, }, @@ -67,7 +67,7 @@ func TestGetEvent(t *testing.T) { Event: event2, }, cacheArgs: test.CacheMockArgs{ - Key: id2, + Key: "seqapi_event_" + id2, Value: string(event2json), }, }, @@ -80,7 +80,7 @@ func TestGetEvent(t *testing.T) { Event: event3, }, cacheArgs: test.CacheMockArgs{ - Key: id3, + Key: "seqapi_event_" + id3, Value: string(event3json), Err: errSomethingWrong, }, @@ -91,7 +91,7 @@ func TestGetEvent(t *testing.T) { Id: id4, }, cacheArgs: test.CacheMockArgs{ - Key: id4, + Key: "seqapi_event_" + id4, Err: errSomethingWrong, }, clientErr: errors.New("client error"), @@ -104,8 +104,10 @@ func TestGetEvent(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: config.SeqAPIGlobalOptions{ - EventsCacheTTL: cacheTTL, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Events: &config.HandlerCache{TTL: cacheTTL}, + }, }, }, } @@ -369,9 +371,11 @@ func TestGetEventWithMasking(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: config.SeqAPIGlobalOptions{ - EventsCacheTTL: cacheTTL, - Masking: tt.maskingCfg, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Events: &config.HandlerCache{TTL: cacheTTL}, + }, + Masking: tt.maskingCfg, }, }, } @@ -379,7 +383,7 @@ func TestGetEventWithMasking(t *testing.T) { ctrl := gomock.NewController(t) cacheMock := mock_cache.NewMockCache(ctrl) cacheArgs := test.CacheMockArgs{ - Key: curEID, + Key: "seqapi_event_" + curEID, Value: string(curEData.eventJson), } if !tt.isCached { diff --git a/internal/api/seqapi/v1/grpc/fetch_async_search_result.go b/internal/api/seqapi/v1/grpc/fetch_async_search_result.go index 3ba0fd4..f96eaa8 100644 --- a/internal/api/seqapi/v1/grpc/fetch_async_search_result.go +++ b/internal/api/seqapi/v1/grpc/fetch_async_search_result.go @@ -15,7 +15,8 @@ import ( ) func (a *API) FetchAsyncSearchResult(ctx context.Context, req *seqapi.FetchAsyncSearchResultRequest) (*seqapi.FetchAsyncSearchResultResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -46,7 +47,7 @@ func (a *API) FetchAsyncSearchResult(ctx context.Context, req *seqapi.FetchAsync return nil, status.Error(codes.InvalidArgument, "invalid search_id") } - resp, err := a.asyncSearches.FetchAsyncSearchResult(ctx, req) + resp, err := asyncSearches.FetchAsyncSearchResult(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/fields.go b/internal/api/seqapi/v1/grpc/fields.go index 84d11b8..281756b 100644 --- a/internal/api/seqapi/v1/grpc/fields.go +++ b/internal/api/seqapi/v1/grpc/fields.go @@ -2,17 +2,55 @@ package grpc import ( "context" + "time" "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) +type fieldsCache struct { + cache cache.Cache +} + +func newFieldsCache(c cache.Cache) *fieldsCache { + return &fieldsCache{cache: c} +} + +func (c *fieldsCache) get(ctx context.Context, key string) ([]*seqapi.Field, bool) { + data, err := c.cache.Get(ctx, key) + if err != nil { + return nil, false + } + + var resp seqapi.GetFieldsResponse + if err := proto.Unmarshal([]byte(data), &resp); err != nil { + logger.Error("can't unmarshal cached fields", zap.Error(err)) + return nil, false + } + + return resp.GetFields(), true +} + +func (c *fieldsCache) set(ctx context.Context, key string, fields []*seqapi.Field, ttl time.Duration) { + data, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: fields}) + if err != nil { + logger.Error("can't marshal fields for cache", zap.Error(err)) + return + } + + if err := c.cache.SetWithTTL(ctx, key, string(data), ttl); err != nil { + logger.Error("can't set fields to cache", zap.Error(err)) + } +} + func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seqapi.GetFieldsResponse, error) { ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_fields") defer span.End() @@ -23,48 +61,43 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq return nil, status.Error(codes.InvalidArgument, err.Error()) } - if a.globalParams.fieldsCache == nil { + if params.fieldsCache == nil { resp, err := params.client.GetFields(ctx, req) if err != nil { return nil, err } - resp.SystemFields = a.globalParams.systemFields - resp.PinnedFields = a.globalParams.pinnedFields + resp.SystemFields = params.systemFields + resp.PinnedFields = params.pinnedFields return resp, nil } - fields, cached, isActual := a.globalParams.fieldsCache.getFields() - if cached && isActual { + if cachedFields, ok := params.fieldsCache.get(ctx, params.fieldsCacheKey); ok { return &seqapi.GetFieldsResponse{ - Fields: fields, - SystemFields: a.globalParams.systemFields, - PinnedFields: a.globalParams.pinnedFields, + Fields: cachedFields, + SystemFields: params.systemFields, + PinnedFields: params.pinnedFields, }, nil } resp, err := params.client.GetFields(ctx, req) if err != nil { - if cached { - logger.Error("can't get fields; use cached fields", zap.Error(err)) - return &seqapi.GetFieldsResponse{ - Fields: fields, - SystemFields: a.globalParams.systemFields, - PinnedFields: a.globalParams.pinnedFields, - }, nil - } - return nil, err } - a.globalParams.fieldsCache.setFields(resp.GetFields()) - resp.SystemFields = a.globalParams.systemFields - resp.PinnedFields = a.globalParams.pinnedFields + params.fieldsCache.set(ctx, params.fieldsCacheKey, resp.GetFields(), params.options.Caches.Fields.TTL) + resp.SystemFields = params.systemFields + resp.PinnedFields = params.pinnedFields return resp, nil } func (a *API) GetPinnedFields(ctx context.Context, _ *seqapi.GetFieldsRequest) (*seqapi.GetFieldsResponse, error) { + env := types.GetEnvKey(ctx) + params, err := a.GetParams(env) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } return &seqapi.GetFieldsResponse{ - Fields: a.globalParams.pinnedFields, + Fields: params.pinnedFields, }, nil } diff --git a/internal/api/seqapi/v1/grpc/fields_test.go b/internal/api/seqapi/v1/grpc/fields_test.go index 0e78bc0..0cdc9a6 100644 --- a/internal/api/seqapi/v1/grpc/fields_test.go +++ b/internal/api/seqapi/v1/grpc/fields_test.go @@ -12,6 +12,8 @@ import ( "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/pkg/cache" + mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -20,7 +22,7 @@ func TestGetFields(t *testing.T) { tests := []struct { name string - cfg config.SeqAPIGlobalOptions + cfg config.SeqAPIOptions seqDBResp *seqapi.GetFieldsResponse wantResp *seqapi.GetFieldsResponse @@ -55,7 +57,7 @@ func TestGetFields(t *testing.T) { }, { name: "ok_with_system_and_pinned_fields", - cfg: config.SeqAPIGlobalOptions{ + cfg: config.SeqAPIOptions{ SystemFields: []config.Field{ {Name: "field1", Type: "keyword"}, {Name: "field2", Type: "text"}, @@ -121,7 +123,7 @@ func TestGetFields(t *testing.T) { SeqDB: seqDbMock, }, Cfg: config.SeqAPI{ - GlobalOptions: tt.cfg, + Options: tt.cfg, }, } @@ -136,66 +138,70 @@ func TestGetFields(t *testing.T) { func TestGetFieldsCached(t *testing.T) { var ( - ttl = 10 * time.Millisecond + ttl = 1 * time.Hour + cacheKey = "seqapi_fields" ) - responses := []*seqapi.GetFieldsResponse{ - { - Fields: []*seqapi.Field{ - { - Name: "n1", - Type: seqapi.FieldType_keyword, - }, - { - Name: "n2", - Type: seqapi.FieldType_text, - }, + + responses := &seqapi.GetFieldsResponse{ + Fields: []*seqapi.Field{ + { + Name: "n1", + Type: seqapi.FieldType_keyword, }, - }, - { - Fields: []*seqapi.Field{ - { - Name: "qwe", - Type: seqapi.FieldType_text, - }, + { + Name: "n2", + Type: seqapi.FieldType_text, }, }, } + cached, err := proto.Marshal(responses) + require.NoError(t, err) + ctrl := gomock.NewController(t) seqDbMock := mock_seqdb.NewMockClient(ctrl) - - for _, r := range responses { - seqDbMock.EXPECT(). - GetFields(gomock.Any(), nil). - Return(proto.Clone(r), nil). - Times(1) - } + cacheMock := mock_cache.NewMockCache(ctrl) + + // first call - cache miss, requested from seqdb and set to cache + cacheMock.EXPECT(). + Get(gomock.Any(), cacheKey). + Return("", cache.ErrNotFound). + Times(1) + seqDbMock.EXPECT(). + GetFields(gomock.Any(), nil). + Return(proto.Clone(responses), nil). + Times(1) + cacheMock.EXPECT(). + SetWithTTL(gomock.Any(), cacheKey, string(cached), ttl). + Return(nil). + Times(1) + + // second call - fields from cache + cacheMock.EXPECT(). + Get(gomock.Any(), cacheKey). + Return(string(cached), nil). + Times(1) seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: config.SeqAPIGlobalOptions{ - FieldsCacheTTL: ttl, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Fields: &config.HandlerCache{TTL: ttl}, + }, }, }, Mocks: test.Mocks{ SeqDB: seqDbMock, + Cache: cacheMock, }, } api := setupTestAPI(seqData) - for _, r := range responses { + for range 2 { resp, err := api.GetFields(context.Background(), nil) require.NoError(t, err) - require.True(t, proto.Equal(r, resp)) - - time.Sleep(ttl / 2) - - resp, err = api.GetFields(context.Background(), nil) - require.NoError(t, err) - require.True(t, proto.Equal(r, resp)) - - time.Sleep(ttl) + require.True(t, proto.Equal(responses, resp)) } } @@ -222,7 +228,7 @@ func TestGetPinnedFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: config.SeqAPIGlobalOptions{ + Options: config.SeqAPIOptions{ PinnedFields: tt.fields, }, }, diff --git a/internal/api/seqapi/v1/grpc/get_async_searches_list.go b/internal/api/seqapi/v1/grpc/get_async_searches_list.go index b639207..47a1fc2 100644 --- a/internal/api/seqapi/v1/grpc/get_async_searches_list.go +++ b/internal/api/seqapi/v1/grpc/get_async_searches_list.go @@ -14,7 +14,8 @@ import ( ) func (a *API) GetAsyncSearchesList(ctx context.Context, req *seqapi.GetAsyncSearchesListRequest) (*seqapi.GetAsyncSearchesListResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -50,7 +51,7 @@ func (a *API) GetAsyncSearchesList(ctx context.Context, req *seqapi.GetAsyncSear return nil, status.Error(codes.InvalidArgument, err.Error()) } - resp, err := a.asyncSearches.GetAsyncSearchesList(ctx, req) + resp, err := asyncSearches.GetAsyncSearchesList(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/get_envs_test.go b/internal/api/seqapi/v1/grpc/get_envs_test.go index 0acda54..9d98f88 100644 --- a/internal/api/seqapi/v1/grpc/get_envs_test.go +++ b/internal/api/seqapi/v1/grpc/get_envs_test.go @@ -21,11 +21,13 @@ func TestGetEnvs(t *testing.T) { name: "single_env", cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: &seqapi.GetEnvsResponse{ @@ -47,47 +49,57 @@ func TestGetEnvs(t *testing.T) { Envs: map[string]config.SeqAPIEnv{ "test": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, "prod": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 150, - MaxExportLimit: 250, - MaxParallelExportRequests: 3, - MaxAggregationsPerRequest: 6, - SeqCLIMaxSearchLimit: 15000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 150, + MaxExportLimit: 250, + MaxParallelExportRequests: 3, + MaxAggregationsPerRequest: 6, + SeqCLIMaxSearchLimit: 15000, + }, }, }, "cluster-10": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 150, - MaxExportLimit: 250, - MaxParallelExportRequests: 3, - MaxAggregationsPerRequest: 6, - SeqCLIMaxSearchLimit: 15000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 150, + MaxExportLimit: 250, + MaxParallelExportRequests: 3, + MaxAggregationsPerRequest: 6, + SeqCLIMaxSearchLimit: 15000, + }, }, }, "cluster-102": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, "cluster-220": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, }, diff --git a/internal/api/seqapi/v1/grpc/limits.go b/internal/api/seqapi/v1/grpc/limits.go index 0a5b702..fb2a66d 100644 --- a/internal/api/seqapi/v1/grpc/limits.go +++ b/internal/api/seqapi/v1/grpc/limits.go @@ -18,10 +18,10 @@ func (a *API) GetLimits(ctx context.Context, _ *seqapi.GetLimitsRequest) (*seqap } return &seqapi.GetLimitsResponse{ - MaxSearchLimit: params.options.MaxSearchLimit, - MaxExportLimit: params.options.MaxExportLimit, - MaxParallelExportRequests: int32(params.options.MaxParallelExportRequests), - MaxAggregationsPerRequest: int32(params.options.MaxAggregationsPerRequest), - SeqCliMaxSearchLimit: int32(params.options.SeqCLIMaxSearchLimit), + MaxSearchLimit: params.options.Limits.MaxSearchLimit, + MaxExportLimit: params.options.Limits.MaxExportLimit, + MaxParallelExportRequests: int32(params.options.Limits.MaxParallelExportRequests), + MaxAggregationsPerRequest: int32(params.options.Limits.MaxAggregationsPerRequest), + SeqCliMaxSearchLimit: int32(params.options.Limits.SeqCLIMaxSearchLimit), }, nil } diff --git a/internal/api/seqapi/v1/grpc/limits_test.go b/internal/api/seqapi/v1/grpc/limits_test.go index 3d1bc4f..dc4f498 100644 --- a/internal/api/seqapi/v1/grpc/limits_test.go +++ b/internal/api/seqapi/v1/grpc/limits_test.go @@ -22,11 +22,13 @@ func TestGetLimits(t *testing.T) { name: "ok", cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: &seqapi.GetLimitsResponse{ diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan.go b/internal/api/seqapi/v1/grpc/logs_lifespan.go index 0bb265e..9648325 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan.go @@ -31,19 +31,19 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ return nil, status.Error(codes.InvalidArgument, err.Error()) } - cacheKey := "logs_lifespan" - if countStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { - count := 0 - count, err = strconv.Atoi(countStr) - if err == nil { - res := time.Duration(count) * lifespan.MeasureUnit - logger.Debug("got logs lifespan from cache") - return &seqapi.GetLogsLifespanResponse{Lifespan: durationpb.New(res)}, nil - } + if params.logsLifespanCache != nil { + if countStr, err := params.logsLifespanCache.Get(ctx, params.logsLifespanCacheKey); err == nil { + count, err := strconv.Atoi(countStr) + if err == nil { + res := time.Duration(count) * lifespan.MeasureUnit + logger.Debug("got logs lifespan from cache") + return &seqapi.GetLogsLifespanResponse{Lifespan: durationpb.New(res)}, nil + } - logger.Error("can't parse logs lifespan", zap.Error(err)) - } else if !errors.Is(err, cache.ErrNotFound) { - logger.Error("can't get logs lifespan from cache", zap.Error(err)) + logger.Error("can't parse logs lifespan", zap.Error(err)) + } else if !errors.Is(err, cache.ErrNotFound) { + logger.Error("can't get logs lifespan from cache", zap.Error(err)) + } } clientStatus, err := params.client.Status(ctx, &seqapi.StatusRequest{}) @@ -58,9 +58,11 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ count := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) res := time.Duration(count) * lifespan.MeasureUnit - err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(count), a.globalParams.logsLifespanCacheTTL) - if err != nil { - logger.Error("can't set logs lifespan to cache", zap.Error(err)) + if params.logsLifespanCache != nil { + err = params.logsLifespanCache.SetWithTTL(ctx, params.logsLifespanCacheKey, strconv.Itoa(count), params.options.Caches.LogsLifespan.TTL) + if err != nil { + logger.Error("can't set logs lifespan to cache", zap.Error(err)) + } } logger.Debug("got logs lifespan from seq-proxy") diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index 18095e7..e71d99f 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -23,7 +23,7 @@ import ( func TestGetLogsLifespan(t *testing.T) { var ( resultStr = "36000" // 10(h) * 60(min/h) * 60(sec/min) - cacheKey = "logs_lifespan" + cacheKey = "seqapi_logs_lifespan" result = 10 * time.Hour cacheTTL = time.Minute ) @@ -106,8 +106,10 @@ func TestGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: config.SeqAPIGlobalOptions{ - LogsLifespanCacheTTL: cacheTTL, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + LogsLifespan: &config.HandlerCache{TTL: cacheTTL}, + }, }, }, } diff --git a/internal/api/seqapi/v1/grpc/search.go b/internal/api/seqapi/v1/grpc/search.go index 7046cc9..943c979 100644 --- a/internal/api/seqapi/v1/grpc/search.go +++ b/internal/api/seqapi/v1/grpc/search.go @@ -87,16 +87,16 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se span.SetAttributes(spanAttributes...) - if err := api_error.CheckSearchLimit(req.Limit, params.options.MaxSearchLimit); err != nil { + if err := api_error.CheckSearchLimit(req.Limit, params.options.Limits.MaxSearchLimit); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - if err := api_error.CheckAggregationsCount(len(req.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(req.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } if err := api_error.CheckForOffsetConflict(req.Offset, req.OffsetId); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - if err := api_error.CheckSearchOffsetLimit(req.Offset, params.options.MaxSearchOffsetLimit); err != nil { + if err := api_error.CheckSearchOffsetLimit(req.Offset, params.options.Limits.MaxSearchOffset); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -106,7 +106,7 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se continue } if err := api_error.CheckAggregationTsInterval(*agg.Interval, fromRaw, toRaw, - params.options.MaxBucketsPerAggregationTs, + params.options.Limits.MaxBucketsPerAggregationTs, ); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -117,16 +117,16 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se return nil, err } - if resp.Total > params.options.MaxSearchTotalLimit { + if resp.Total > params.options.Limits.MaxSearchTotal { resp.Error = &seqapi.Error{ Code: seqapi.ErrorCode_ERROR_CODE_QUERY_TOO_HEAVY, Message: api_error.ErrQueryTooHeavy.Error(), } } - if a.globalParams.masker != nil { + if params.masker != nil { for _, e := range resp.Events { - a.globalParams.masker.Mask(e.Data) + params.masker.Mask(e.Data) } } diff --git a/internal/api/seqapi/v1/grpc/search_test.go b/internal/api/seqapi/v1/grpc/search_test.go index 148b59e..273f06e 100644 --- a/internal/api/seqapi/v1/grpc/search_test.go +++ b/internal/api/seqapi/v1/grpc/search_test.go @@ -63,8 +63,10 @@ func TestSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 5, + }, }, }), }, @@ -82,7 +84,9 @@ func TestSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }, apiErr: true, @@ -99,8 +103,10 @@ func TestSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 2, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 2, + }, }, }), apiErr: true, @@ -116,8 +122,10 @@ func TestSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxSearchOffset: 10, + }, }, }), apiErr: true, @@ -146,8 +154,10 @@ func TestSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchTotalLimit: int64(limit), + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxSearchTotal: int64(limit), + }, }, }), }, @@ -162,7 +172,9 @@ func TestSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), clientErr: errSomethingWrong, diff --git a/internal/api/seqapi/v1/grpc/start_async_search.go b/internal/api/seqapi/v1/grpc/start_async_search.go index 86641ed..d55c316 100644 --- a/internal/api/seqapi/v1/grpc/start_async_search.go +++ b/internal/api/seqapi/v1/grpc/start_async_search.go @@ -15,7 +15,8 @@ import ( ) func (a *API) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSearchRequest) (*seqapi.StartAsyncSearchResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -64,7 +65,7 @@ func (a *API) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSearch span.SetAttributes(spanAttributes...) - resp, err := a.asyncSearches.StartAsyncSearch(ctx, req) + resp, err := asyncSearches.StartAsyncSearch(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index ff0d6ab..6987adc 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -18,21 +18,32 @@ var ( ) func setupTestAPI(data test.APITestData) *API { - if len(data.Cfg.Envs) == 0 && data.Cfg.SeqDBID == "" { - data.Cfg.SeqDBID = config.DefaultSeqDBClientID - } - - seqDBClients := make(map[string]seqdb.Client) - seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB + seqDBClients := make(map[string]seqdb.Client, max(1, len(data.Cfg.Envs))) + asyncSearhes := make(map[string]asyncsearches.Service, max(1, len(data.Cfg.Envs))) - for _, envConfig := range data.Cfg.Envs { + if len(data.Cfg.Envs) == 0 { + seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB + asyncSearhes[""] = data.Mocks.AsyncSearchesSvc + } + for envName, envConfig := range data.Cfg.Envs { seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB + asyncSearhes[envName] = data.Mocks.AsyncSearchesSvc } - var asyncSvc asyncsearches.Service - if data.Mocks.AsyncSearchesSvc != nil { - asyncSvc = data.Mocks.AsyncSearchesSvc + if data.Mocks.AsyncSearchesSvc == nil { + asyncSearhes = nil + } + + api := New(&data.Cfg, seqDBClients, asyncSearhes) + + if data.Mocks.Cache != nil { + for env, params := range api.paramsByEnv { + params.eventsCache = data.Mocks.Cache + params.logsLifespanCache = data.Mocks.Cache + params.fieldsCache = newFieldsCache(data.Mocks.Cache) + api.paramsByEnv[env] = params + } } - return New(&data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) + return api } diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index 93e2cf4..0fc12f3 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -28,55 +28,49 @@ type apiParams struct { pinnedFields fields systemFields fields exportLimiter *tokenlimiter.Limiter - eventCacheKeyPrefix string + eventsCache cache.Cache + logsLifespanCache cache.Cache + fieldsCache *fieldsCache + eventsCacheKeyPrefix string fieldsCacheKey string logsLifespanCacheKey string } type API struct { - config *config.SeqAPI - paramsByEnv map[string]apiParams - inmemWithRedisCache cache.Cache - redisCache cache.Cache - fieldsCache *fieldsCache - nowFn func() time.Time - asyncSearches asyncsearches.Service - envsResponse getEnvsResponse + config *config.SeqAPI + paramsByEnv map[string]apiParams + nowFn func() time.Time + asyncSearches map[string]asyncsearches.Service + envsResponse getEnvsResponse } -func New( - cfg *config.SeqAPI, - seqDBСlients map[string]seqdb.Client, - inmemWithRedisCache cache.Cache, - redisCache cache.Cache, - fieldsCache cache.Cache, - asyncSearches asyncsearches.Service, -) *API { +func New(cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, asyncSearches map[string]asyncsearches.Service) *API { paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { for envName, envConfig := range cfg.Envs { paramsByEnv[envName] = newAPIParams(envName, seqDBСlients[envConfig.SeqDBID], envConfig.Options) } } else { - paramsByEnv[""] = newAPIParams("", seqDBСlients[cfg.SeqDBID], &cfg.Options) + var client seqdb.Client + for _, c := range seqDBСlients { + client = c + } + paramsByEnv[""] = newAPIParams("", client, &cfg.Options) } return &API{ - config: cfg, - paramsByEnv: paramsByEnv, - inmemWithRedisCache: inmemWithRedisCache, - redisCache: redisCache, - fieldsCache: newFieldsCache(fieldsCache), - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + config: cfg, + paramsByEnv: paramsByEnv, + nowFn: time.Now, + asyncSearches: asyncSearches, + envsResponse: parseEnvs(cfg), } } func newAPIParams(env string, client seqdb.Client, options *config.SeqAPIOptions) apiParams { masker, err := mask.New(options.Masking) if err != nil { - logger.Fatal("failed to init masking", zap.Error(err)) + logger.Fatal("failed to init masking", zap.String("env", env), zap.Error(err)) } // for export @@ -84,7 +78,25 @@ func newAPIParams(env string, client seqdb.Client, options *config.SeqAPIOptions client.WithMasking(masker) } - cacheKeyPrefix := "seqapi_" + env + "_" + var eventsCache, logsLifespanCache cache.Cache + if options.Caches.Events != nil { + eventsCache, _ = cache.ByID(options.Caches.Events.ID) + } + if options.Caches.LogsLifespan != nil { + logsLifespanCache, _ = cache.ByID(options.Caches.LogsLifespan.ID) + } + + var fc *fieldsCache + if options.Caches.Fields != nil { + if c, ok := cache.ByID(options.Caches.Fields.ID); ok { + fc = newFieldsCache(c) + } + } + + cacheKeyPrefix := "seqapi_" + if env != "" { + cacheKeyPrefix += env + "_" + } return apiParams{ client: client, @@ -93,7 +105,10 @@ func newAPIParams(env string, client seqdb.Client, options *config.SeqAPIOptions pinnedFields: parseFields(options.PinnedFields), systemFields: parseFields(options.SystemFields), exportLimiter: tokenlimiter.New(options.Limits.MaxParallelExportRequests), - eventCacheKeyPrefix: cacheKeyPrefix + "event_", + eventsCache: eventsCache, + logsLifespanCache: logsLifespanCache, + fieldsCache: fc, + eventsCacheKeyPrefix: cacheKeyPrefix + "event_", fieldsCacheKey: cacheKeyPrefix + "fields", logsLifespanCacheKey: cacheKeyPrefix + "logs_lifespan", } diff --git a/internal/api/seqapi/v1/http/cancel_async_search.go b/internal/api/seqapi/v1/http/cancel_async_search.go index b07776d..f2c2a91 100644 --- a/internal/api/seqapi/v1/http/cancel_async_search.go +++ b/internal/api/seqapi/v1/http/cancel_async_search.go @@ -23,31 +23,40 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveCancelAsyncSearch(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_cancel_async_search") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_cancel_async_search") - defer span.End() - searchID := chi.URLParam(r, "id") - + // я бы в service вынес проверочку и попробовал перейти на стандартную либу uuid(она вроде с этой версии появилось) + // https://go.dev/doc/go1.27#uuid такая вот фигня if err := checkUUID(searchID); err != nil { wr.Error(err, http.StatusBadRequest) return } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "search_id", Value: attribute.StringValue(searchID), }, - ) + } + + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + + span.SetAttributes(attributes...) - _, err := a.asyncSearches.CancelAsyncSearch(ctx, &seqapi.CancelAsyncSearchRequest{SearchId: searchID}) + _, err := asyncSearches.CancelAsyncSearch(ctx, &seqapi.CancelAsyncSearchRequest{SearchId: searchID}) if err != nil { status := http.StatusInternalServerError if errors.Is(err, types.ErrPermissionDenied) { diff --git a/internal/api/seqapi/v1/http/delete_async_search.go b/internal/api/seqapi/v1/http/delete_async_search.go index 28e64a7..cd9b6dc 100644 --- a/internal/api/seqapi/v1/http/delete_async_search.go +++ b/internal/api/seqapi/v1/http/delete_async_search.go @@ -23,31 +23,39 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveDeleteAsyncSearch(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_delete_async_search") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_delete_async_search") - defer span.End() - searchID := chi.URLParam(r, "id") - + // в cancel_async описал, чтобы я с этой проверкой сделал if err := checkUUID(searchID); err != nil { wr.Error(err, http.StatusBadRequest) return } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "search_id", Value: attribute.StringValue(searchID), }, - ) + } + + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + + span.SetAttributes(attributes...) - _, err := a.asyncSearches.DeleteAsyncSearch(ctx, &seqapi.DeleteAsyncSearchRequest{SearchId: searchID}) + _, err := asyncSearches.DeleteAsyncSearch(ctx, &seqapi.DeleteAsyncSearchRequest{SearchId: searchID}) if err != nil { status := http.StatusInternalServerError if errors.Is(err, types.ErrPermissionDenied) { diff --git a/internal/api/seqapi/v1/http/events.go b/internal/api/seqapi/v1/http/events.go index 26fffc0..c9c725b 100644 --- a/internal/api/seqapi/v1/http/events.go +++ b/internal/api/seqapi/v1/http/events.go @@ -31,10 +31,8 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - - id := chi.URLParam(r, "id") - env := types.GetEnvKey(ctx) + id := chi.URLParam(r, "id") attributes := []attribute.KeyValue{ { @@ -55,22 +53,22 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { return } - cacheKey := params.eventCacheKeyPrefix + id - if cached, err := a.inmemWithRedisCache.Get(ctx, cacheKey); err == nil { - e := &seqapi.Event{} - if err = proto.Unmarshal([]byte(cached), e); err == nil { - if params.masker != nil { - params.masker.Mask(e.Data) + cacheKey := params.eventsCacheKeyPrefix + id // как лучше с ним поступить не придумал + if params.eventsCache != nil { + if cached, err := params.eventsCache.Get(ctx, cacheKey); err == nil { + e := &seqapi.Event{} + if err = proto.Unmarshal([]byte(cached), e); err == nil { + if params.masker != nil { + params.masker.Mask(e.Data) + } + wr.WriteJson(getEventResponseFromProto(&seqapi.GetEventResponse{Event: e})) + return } - wr.WriteJson(getEventResponseFromProto(&seqapi.GetEventResponse{Event: e})) - return + logger.Error("failed to unmarshal cached event proto", zap.String("id", id), zap.Error(err)) } - logger.Error("failed to unmarshal cached event proto", zap.String("id", id), zap.Error(err)) } - resp, err := params.client.GetEvent(ctx, &seqapi.GetEventRequest{ - Id: id, - }) + resp, err := params.client.GetEvent(ctx, &seqapi.GetEventRequest{Id: id}) if err != nil { wr.Error(err, http.StatusInternalServerError) return @@ -80,10 +78,12 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { params.masker.Mask(resp.Event.Data) } - if data, err := proto.Marshal(resp.Event); err == nil { - _ = a.inmemWithRedisCache.SetWithTTL(ctx, cacheKey, string(data), params.options.Caches.Events.TTL) - } else { - logger.Error("failed to marshal event proto for caching", zap.String("id", id), zap.Error(err)) + if params.eventsCache != nil { + if data, err := proto.Marshal(resp.Event); err == nil { + _ = params.eventsCache.SetWithTTL(ctx, cacheKey, string(data), params.options.Caches.Events.TTL) + } else { + logger.Error("failed to marshal event proto for caching", zap.String("id", id), zap.Error(err)) + } } eventResp := getEventResponseFromProto(resp) diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index bed5b93..4878b8b 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -57,7 +57,7 @@ func TestServeGetEvent(t *testing.T) { id: id1, want: getEventResponse{Event: eventFromProto(event1)}, cacheArgs: test.CacheMockArgs{ - Key: id1, + Key: "seqapi_event_" + id1, Value: string(event1json), Err: errSomethingWrong, }, @@ -75,7 +75,7 @@ func TestServeGetEvent(t *testing.T) { id: id2, want: getEventResponse{Event: eventFromProto(event2)}, cacheArgs: test.CacheMockArgs{ - Key: id2, + Key: "seqapi_event_" + id2, Value: string(event2json), }, }, @@ -84,7 +84,7 @@ func TestServeGetEvent(t *testing.T) { id: id3, want: getEventResponse{Event: eventFromProto(event3)}, cacheArgs: test.CacheMockArgs{ - Key: id3, + Key: "seqapi_event_" + id3, Value: string(event3json), Err: errSomethingWrong, }, @@ -102,7 +102,7 @@ func TestServeGetEvent(t *testing.T) { id: id4, wantErr: true, cacheArgs: test.CacheMockArgs{ - Key: id4, + Key: "seqapi_event_" + id4, Err: errSomethingWrong, }, mockArgs: &mockArgs{ @@ -121,8 +121,10 @@ func TestServeGetEvent(t *testing.T) { ctrl := gomock.NewController(t) seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: config.SeqAPIGlobalOptions{ - EventsCacheTTL: cacheTTL, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Events: &config.HandlerCache{TTL: cacheTTL}, + }, }, }, } @@ -381,16 +383,18 @@ func TestGetEventWithMasking(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - GlobalOptions: config.SeqAPIGlobalOptions{ - EventsCacheTTL: cacheTTL, - Masking: tt.maskingCfg, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Events: &config.HandlerCache{TTL: cacheTTL}, + }, + Masking: tt.maskingCfg, }, }, } cacheMock := mock_cache.NewMockCache(ctrl) cacheArgs := test.CacheMockArgs{ - Key: curEID, + Key: "seqapi_event_" + curEID, Value: string(curEData.eventJson), } if !tt.isCached { diff --git a/internal/api/seqapi/v1/http/export.go b/internal/api/seqapi/v1/http/export.go index 1b929af..9e10401 100644 --- a/internal/api/seqapi/v1/http/export.go +++ b/internal/api/seqapi/v1/http/export.go @@ -32,7 +32,6 @@ func (a *API) serveExport(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - env := types.GetEnvKey(ctx) userStr := "_" diff --git a/internal/api/seqapi/v1/http/export_async_search.go b/internal/api/seqapi/v1/http/export_async_search.go index 1afc559..f71a506 100644 --- a/internal/api/seqapi/v1/http/export_async_search.go +++ b/internal/api/seqapi/v1/http/export_async_search.go @@ -26,18 +26,18 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_export_async_search_http") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_export_async_search_http") - defer span.End() - - env := types.GetEnvKey(ctx) - userStr := "_" if userName, err := types.GetUserKey(ctx); err == nil { userStr = userName @@ -123,7 +123,7 @@ func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { return } - if err := a.asyncSearches.ExportAsyncSearch(ctx, httpReq.toProto(), cw); err != nil { + if err := asyncSearches.ExportAsyncSearch(ctx, httpReq.toProto(), cw); err != nil { wr.Error(err, http.StatusInternalServerError) return } diff --git a/internal/api/seqapi/v1/http/fetch_async_search_result.go b/internal/api/seqapi/v1/http/fetch_async_search_result.go index fb09d5a..24b0a13 100644 --- a/internal/api/seqapi/v1/http/fetch_async_search_result.go +++ b/internal/api/seqapi/v1/http/fetch_async_search_result.go @@ -25,16 +25,18 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveFetchAsyncSearchResult(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_fetch_async_search_result") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_fetch_async_search_result") - defer span.End() - var httpReq fetchAsyncSearchResultRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) @@ -59,6 +61,11 @@ func (a *API) serveFetchAsyncSearchResult(w http.ResponseWriter, r *http.Request Value: attribute.StringValue(string(httpReq.Order)), }, } + + if env != "" { + spanAttributes = append(spanAttributes, attribute.String("env", env)) + } + span.SetAttributes(spanAttributes...) if err := checkUUID(httpReq.SearchID); err != nil { @@ -71,7 +78,7 @@ func (a *API) serveFetchAsyncSearchResult(w http.ResponseWriter, r *http.Request return } - resp, err := a.asyncSearches.FetchAsyncSearchResult(ctx, httpReq.toProto()) + resp, err := asyncSearches.FetchAsyncSearchResult(ctx, httpReq.toProto()) if err != nil { wr.Error(err, http.StatusInternalServerError) return diff --git a/internal/api/seqapi/v1/http/fields.go b/internal/api/seqapi/v1/http/fields.go index 6924938..f89a9ab 100644 --- a/internal/api/seqapi/v1/http/fields.go +++ b/internal/api/seqapi/v1/http/fields.go @@ -73,7 +73,7 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - if a.fieldsCache == nil { + if params.fieldsCache == nil { resp, err := params.client.GetFields(ctx, &seqapi.GetFieldsRequest{}) if err != nil { wr.Error(err, http.StatusInternalServerError) @@ -89,7 +89,7 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - if cachedFields, ok := a.fieldsCache.get(ctx, params.fieldsCacheKey); ok { + if cachedFields, ok := params.fieldsCache.get(ctx, params.fieldsCacheKey); ok { res := getFieldsResponse{ Fields: fieldsFromProto(cachedFields), SystemFields: params.systemFields, @@ -105,7 +105,7 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - a.fieldsCache.set(ctx, params.fieldsCacheKey, resp.GetFields(), params.options.Caches.Fields.TTL) + params.fieldsCache.set(ctx, params.fieldsCacheKey, resp.GetFields(), params.options.Caches.Fields.TTL) res := getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), SystemFields: params.systemFields, diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index 3c29e03..5d86035 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -1,21 +1,24 @@ package http import ( - "encoding/json" "net/http" "testing" "time" - "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" "github.com/ozontech/seq-ui/internal/app/config/v2" + mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" + "github.com/stretchr/testify/require" ) +var fieldsCacheKey = "seqapi_fields" + func TestServeGetFields(t *testing.T) { f := fields{ {Name: "test_name1", Type: "keyword"}, @@ -26,11 +29,8 @@ func TestServeGetFields(t *testing.T) { {Name: "test_name2", Type: seqapi.FieldType_text}, } - marshal := func(data any) []byte { - raw, err := json.Marshal(data) - require.NoError(t, err) - return raw - } + cachedFields, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: fAPI}) + require.NoError(t, err) type mockArgs struct { resp *seqapi.GetFieldsResponse @@ -93,10 +93,10 @@ func TestServeGetFields(t *testing.T) { want: getFieldsResponse{ Fields: f, }, - cacheData: marshal(getFieldsResponse{Fields: f}), + cacheData: cachedFields, cfg: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Fields: config.HandlerCache{ + Fields: &config.HandlerCache{ TTL: time.Hour, }, }, @@ -125,6 +125,15 @@ func TestServeGetFields(t *testing.T) { seqDbMock := mock_seqdb.NewMockClient(ctrl) seqData.Mocks.SeqDB = seqDbMock + if tt.cacheData != nil { + cacheMock := mock_cache.NewMockCache(ctrl) + cacheMock.EXPECT(). + Get(gomock.Any(), fieldsCacheKey). + Return(string(tt.cacheData), nil). + Times(1) + seqData.Mocks.Cache = cacheMock + } + api := setupTestAPI(seqData) if tt.mockArgs != nil { @@ -133,9 +142,6 @@ func TestServeGetFields(t *testing.T) { Return(tt.mockArgs.resp, tt.mockArgs.err). Times(1) } - if tt.cacheData != nil && tt.cfg.Caches.Fields.TTL > 0 { - api.fieldsCache.set(tt.cacheData) - } httputil.DoTestHTTPEx(t, httputil.TestDataHTTPEx[struct{}, getFieldsResponse]{ Method: http.MethodGet, diff --git a/internal/api/seqapi/v1/http/get_async_searches_list.go b/internal/api/seqapi/v1/http/get_async_searches_list.go index 8e64d7c..a0b0c2c 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list.go @@ -25,16 +25,18 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_async_searches_list") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_async_searches_list") - defer span.End() - var httpReq getAsyncSearchesListRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) @@ -63,6 +65,9 @@ func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) Value: attribute.StringValue(*httpReq.Owner), }) } + if env != "" { + spanAttributes = append(spanAttributes, attribute.String("env", env)) + } span.SetAttributes(spanAttributes...) @@ -77,7 +82,7 @@ func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) return } - resp, err := a.asyncSearches.GetAsyncSearchesList(ctx, parsedReq) + resp, err := asyncSearches.GetAsyncSearchesList(ctx, parsedReq) if err != nil { wr.Error(err, http.StatusInternalServerError) return diff --git a/internal/api/seqapi/v1/http/limits.go b/internal/api/seqapi/v1/http/limits.go index f5bace7..6d04d8b 100644 --- a/internal/api/seqapi/v1/http/limits.go +++ b/internal/api/seqapi/v1/http/limits.go @@ -30,8 +30,8 @@ func (a *API) serveGetLimits(w http.ResponseWriter, r *http.Request) { } httputil.NewWriter(w).WriteJson(getLimitsResponse{ - MaxSearchLimit: params.options.Limits.MaxSearchLimit, - MaxExportLimit: params.options.Limits.MaxExportLimit, + MaxSearchLimit: int(params.options.Limits.MaxSearchLimit), + MaxExportLimit: int(params.options.Limits.MaxExportLimit), MaxParallelExportRequests: params.options.Limits.MaxParallelExportRequests, MaxAggregationsPerRequest: params.options.Limits.MaxAggregationsPerRequest, SeqCliMaxSearchLimit: params.options.Limits.SeqCLIMaxSearchLimit, diff --git a/internal/api/seqapi/v1/http/logs_lifespan.go b/internal/api/seqapi/v1/http/logs_lifespan.go index 0901bf0..0f55aa8 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan.go +++ b/internal/api/seqapi/v1/http/logs_lifespan.go @@ -38,18 +38,19 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { return } - if resStr, err := a.redisCache.Get(ctx, params.logsLifespanCacheKey); err == nil { - res := 0 - res, err = strconv.Atoi(resStr) - if err == nil { - logger.Debug("got logs lifespan from cache") - wr.WriteJson(getLogsLifespanResponse{Lifespan: res}) - return - } + if params.logsLifespanCache != nil { + if resStr, err := params.logsLifespanCache.Get(ctx, params.logsLifespanCacheKey); err == nil { + res, err := strconv.Atoi(resStr) + if err == nil { + logger.Debug("got logs lifespan from cache") + wr.WriteJson(getLogsLifespanResponse{Lifespan: res}) + return + } - logger.Error("can't parse logs lifespan", zap.Error(err)) - } else if !errors.Is(err, cache.ErrNotFound) { - logger.Error("can't get logs lifespan from cache", zap.Error(err)) + logger.Error("can't parse logs lifespan", zap.Error(err)) + } else if !errors.Is(err, cache.ErrNotFound) { + logger.Error("can't get logs lifespan from cache", zap.Error(err)) + } } clientStatus, err := params.client.Status(ctx, &seqapi.StatusRequest{}) @@ -65,12 +66,14 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { res := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) - if err = a.redisCache.SetWithTTL( - ctx, - params.logsLifespanCacheKey, - strconv.Itoa(res), - params.options.Caches.LogsLifespan.TTL); err != nil { - logger.Error("can't set logs lifespan to cache", zap.Error(err)) + if params.logsLifespanCache != nil { + if err = params.logsLifespanCache.SetWithTTL( + ctx, + params.logsLifespanCacheKey, + strconv.Itoa(res), + params.options.Caches.LogsLifespan.TTL); err != nil { + logger.Error("can't set logs lifespan to cache", zap.Error(err)) + } } logger.Debug("got logs lifespan from seq-proxy") diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index 1b9e300..616cce6 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -21,8 +21,8 @@ import ( func TestServeGetLogsLifespan(t *testing.T) { var ( + cacheKey = "seqapi_logs_lifespan" resultStr = "36000" // 10(h) * 60(min/h) * 60(sec/min) - cacheKey = "logs_lifespan" result = 10 * time.Hour cacheTTL = time.Minute ) @@ -105,7 +105,7 @@ func TestServeGetLogsLifespan(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - LogsLifespan: config.HandlerCache{ + LogsLifespan: &config.HandlerCache{ TTL: cacheTTL, }, }, diff --git a/internal/api/seqapi/v1/http/search.go b/internal/api/seqapi/v1/http/search.go index ad20c6f..5e330fc 100644 --- a/internal/api/seqapi/v1/http/search.go +++ b/internal/api/seqapi/v1/http/search.go @@ -133,7 +133,7 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { return } - if resp.Total > int64(params.options.Limits.MaxSearchTotal) { + if resp.Total > params.options.Limits.MaxSearchTotal { resp.Error = &seqapi.Error{ Code: seqapi.ErrorCode_ERROR_CODE_QUERY_TOO_HEAVY, Message: api_error.ErrQueryTooHeavy.Error(), @@ -170,8 +170,8 @@ type searchRequest struct { Query string `json:"query"` From time.Time `json:"from" format:"date-time"` To time.Time `json:"to" format:"date-time"` - Limit int `json:"limit"` - Offset int `json:"offset"` + Limit int32 `json:"limit"` + Offset int32 `json:"offset"` WithTotal bool `json:"withTotal"` Aggregations aggregationQueries `json:"aggregations"` Histogram struct { diff --git a/internal/api/seqapi/v1/http/search_test.go b/internal/api/seqapi/v1/http/search_test.go index b9b308d..fb2806f 100644 --- a/internal/api/seqapi/v1/http/search_test.go +++ b/internal/api/seqapi/v1/http/search_test.go @@ -90,7 +90,9 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -127,7 +129,9 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -166,7 +170,9 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -214,7 +220,9 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -258,7 +266,9 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -292,7 +302,9 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -333,7 +345,9 @@ func TestServeSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }, wantErr: true, @@ -349,8 +363,10 @@ func TestServeSearch(t *testing.T) { }, cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 2, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 2, + }, }, }, wantErr: true, @@ -366,8 +382,10 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxSearchOffset: 10, + }, }, }), wantErr: true, @@ -389,8 +407,10 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxSearchOffset: 10, + }, }, }), mockArgs: &mockArgs{ @@ -422,7 +442,9 @@ func TestServeSearch(t *testing.T) { }, cfg: test.SetCfgDefaults(config.SeqAPI{ Options: config.SeqAPIOptions{ - MaxSearchLimit: 5, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), wantErr: true, diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index e124ff3..3602923 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -27,17 +27,18 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_start_async_search") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_start_async_search") - defer span.End() - - env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { httputil.NewWriter(w).Error(err, http.StatusBadRequest) @@ -107,10 +108,13 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { Value: attribute.StringValue(string(aggsRaw)), }) } + if env != "" { + spanAttributes = append(spanAttributes, attribute.String("env", env)) + } span.SetAttributes(spanAttributes...) - resp, err := a.asyncSearches.StartAsyncSearch(ctx, httpReq.toProto(parsedRetention)) + resp, err := asyncSearches.StartAsyncSearch(ctx, httpReq.toProto(parsedRetention)) if err != nil { wr.Error(err, http.StatusInternalServerError) return diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 4e727af..43dadde 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -23,23 +23,34 @@ var ( ) func setupTestAPI(data test.APITestData) *API { - if len(data.Cfg.Envs) == 0 && data.Cfg.SeqDBID == "" { - data.Cfg.SeqDBID = config.DefaultSeqDBClientID - } - - seqDBClients := make(map[string]seqdb.Client) - seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB + seqDBClients := make(map[string]seqdb.Client, max(1, len(data.Cfg.Envs))) + asyncSearhes := make(map[string]asyncsearches.Service, max(1, len(data.Cfg.Envs))) - for _, envConfig := range data.Cfg.Envs { + if len(data.Cfg.Envs) == 0 { + seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB + asyncSearhes[""] = data.Mocks.AsyncSearchesSvc + } + for envName, envConfig := range data.Cfg.Envs { seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB + asyncSearhes[envName] = data.Mocks.AsyncSearchesSvc } - var asyncSvc asyncsearches.Service - if data.Mocks.AsyncSearchesSvc != nil { - asyncSvc = data.Mocks.AsyncSearchesSvc + if data.Mocks.AsyncSearchesSvc == nil { + asyncSearhes = nil + } + + api := New(&data.Cfg, seqDBClients, asyncSearhes) + + if data.Mocks.Cache != nil { + for env, params := range api.paramsByEnv { + params.eventsCache = data.Mocks.Cache + params.logsLifespanCache = data.Mocks.Cache + params.fieldsCache = newFieldsCache(data.Mocks.Cache) + api.paramsByEnv[env] = params + } } - return New(&data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) + return api } func withQueryParamID(h http.HandlerFunc, id string) http.HandlerFunc { diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index 753f1c7..d565be9 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -1,13 +1,15 @@ package seqapi_v1 import ( + "context" + "github.com/go-chi/chi/v5" grpc_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/grpc" http_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/http" "github.com/ozontech/seq-ui/internal/app/config/v2" - "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" + "github.com/ozontech/seq-ui/internal/pkg/repository" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) @@ -17,16 +19,17 @@ type SeqAPI struct { } func New( + ctx context.Context, cfg *config.SeqAPI, - seqDB map[string]seqdb.Client, - inmemWithRedisCache cache.Cache, - redisCache cache.Cache, - fieldsCache cache.Cache, - asyncSearches asyncsearches.Service, + seqDBClients map[string]seqdb.Client, + asyncSearhCfg *config.AsyncSearch, + asyncSearchesRepo repository.AsyncSearches, ) *SeqAPI { + asyncSearches := newAsyncSearches(ctx, asyncSearhCfg, asyncSearchesRepo, seqDBClients) + return &SeqAPI{ - grpcAPI: grpc_api.New(cfg, seqDB, inmemWithRedisCache, redisCache, fieldsCache, asyncSearches), - httpAPI: http_api.New(cfg, seqDB, inmemWithRedisCache, redisCache, fieldsCache, asyncSearches), + grpcAPI: grpc_api.New(cfg, seqDBClients, asyncSearches), + httpAPI: http_api.New(cfg, seqDBClients, asyncSearches), } } @@ -37,3 +40,25 @@ func (s *SeqAPI) GRPCServer() *grpc_api.API { func (s *SeqAPI) HTTPRouter() chi.Router { return s.httpAPI.Router() } + +func newAsyncSearches( + ctx context.Context, + cfg *config.AsyncSearch, + repo repository.AsyncSearches, + seqDBClients map[string]seqdb.Client, +) map[string]asyncsearches.Service { + services := make(map[string]asyncsearches.Service, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envCfg := range cfg.Envs { + services[envName] = asyncsearches.New(ctx, repo, seqDBClients[envCfg.SeqDBID], envCfg.Options, envName) + } + } else { + var client seqdb.Client + for _, c := range seqDBClients { + client = c + } + services[""] = asyncsearches.New(ctx, repo, client, &cfg.Options, "") + } + + return services +} From 17c74f5db42ec0ac6c9c098b0f705586389a7829 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 14 Oct 2026 23:33:24 +0300 Subject: [PATCH 22/29] take two: update cmd, api/seqapi-async, app/config-types, pkg/repository-service, migration --- cmd/seq-ui/main.go | 84 ++++++++----------- internal/app/config/v2/config.go | 16 ++-- internal/app/types/async_searches.go | 16 ++++ .../pkg/repository/async_searches_repo.go | 40 ++++----- internal/pkg/repository/repository.go | 6 +- .../pkg/service/async_searches/service.go | 32 +++++-- ...er_table_async_searches_add_column_env.sql | 9 ++ 7 files changed, 113 insertions(+), 90 deletions(-) create mode 100644 migration/12_alter_table_async_searches_add_column_env.sql diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index bf6835a..8f9d0b1 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -20,7 +20,6 @@ import ( "github.com/ozontech/seq-ui/internal/api" admin_v1 "github.com/ozontech/seq-ui/internal/api/admin/v1" dashboards_v1 "github.com/ozontech/seq-ui/internal/api/dashboards/v1" - errorgroups_v1 "github.com/ozontech/seq-ui/internal/api/errorgroups/v1" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" "github.com/ozontech/seq-ui/internal/app/config/loader" @@ -29,12 +28,7 @@ import ( "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/repository" - "github.com/ozontech/seq-ui/internal/pkg/service/admin" - asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" "github.com/ozontech/seq-ui/internal/pkg/service/dashboards" - "github.com/ozontech/seq-ui/internal/pkg/service/massexport" - "github.com/ozontech/seq-ui/internal/pkg/service/massexport/filestore" - "github.com/ozontech/seq-ui/internal/pkg/service/massexport/sessionstore" "github.com/ozontech/seq-ui/internal/pkg/service/profiles" "github.com/ozontech/seq-ui/internal/pkg/service/userprofile" "github.com/ozontech/seq-ui/logger" @@ -140,10 +134,10 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { // } var ( - adminV1 *admin_v1.Admin - asyncSearchesSvc asyncsearches.Service - userProfileV1 *userprofile_v1.UserProfile - dashboardsV1 *dashboards_v1.Dashboards + adminV1 *admin_v1.Admin + asyncSearchesRepo repository.AsyncSearches + userProfileV1 *userprofile_v1.UserProfile + dashboardsV1 *dashboards_v1.Dashboards ) if db != nil { repo := repository.New(db, cfg.DB.RequestTimeout) @@ -158,32 +152,28 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { if cfg.Handlers.AsyncSearch != nil { logger.Info("initializing async search") - asyncSearchesSvc = asyncsearches.New(ctx, repo, seqDBClients, *cfg.Handlers.AsyncSearch) + asyncSearchesRepo = repo.AsyncSearches } - if cfg.Handlers.Admin != nil { - logger.Info("initializing admin") - adminSvc := admin.New(repo.Admin, repo.UserProfiles, adminCache, cfg.Handlers.Admin) - adminV1 = admin_v1.New(adminSvc) - } + // if cfg.Handlers.Admin != nil { + // logger.Info("initializing admin") + // adminSvc := admin.New(repo.Admin, repo.UserProfiles, adminCache, cfg.Handlers.Admin) + // adminV1 = admin_v1.New(adminSvc) + // } } - logger.Info("initializing seqapi") - seqApiV1 := seqapi_v1.New( - &cfg.Handlers.SeqAPI, - seqDBClients, - caches[seqAPICaches.Events.ID], - caches[seqAPICaches.LogsLifespan.ID], - caches[seqAPICaches.Fields.ID], - asyncSearchesSvc, - ) - - var errorGroupsV1 *errorgroups_v1.ErrorGroups - if cfg.Handlers.ErrorGroups != nil { - logger.Info("initializing error groups") - errorGroupsV1 = errorgroups_v1.New(cfg.Handlers.ErrorGroups, &cfg.Clients, chClients) + var seqApiV1 *seqapi_v1.SeqAPI + if cfg.Handlers.SeqAPI != nil { + logger.Info("initializing seqapi") + seqApiV1 = seqapi_v1.New(ctx, cfg.Handlers.SeqAPI, seqDBClients, cfg.Handlers.AsyncSearch, asyncSearchesRepo) } + // var errorGroupsV1 *errorgroups_v1.ErrorGroups + // if cfg.Handlers.ErrorGroups != nil { + // logger.Info("initializing error groups") + // errorGroupsV1 = errorgroups_v1.New(cfg.Handlers.ErrorGroups, &cfg.Clients, chClients) + // } + return api.NewRegistrar(adminV1, seqApiV1, userProfileV1, dashboardsV1, massExportV1, errorGroupsV1) } @@ -201,7 +191,7 @@ func initSeqDBClients(ctx context.Context, cfg *config.Config) (map[string]seqdb } func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *config.SeqAPI) (seqdb.Client, error) { - var maxSearchLimit int + var maxSearchLimit int32 if seqAPI != nil { if len(seqAPI.Envs) == 0 { maxSearchLimit = seqAPI.Options.Limits.MaxSearchLimit @@ -215,7 +205,7 @@ func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *con } } - clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * maxSearchLimit + clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * int(maxSearchLimit) clientMaxRecvMsgSize = max(clientMaxRecvMsgSize, defaultClientMaxRecvMsgSize) clientParams := seqdb.ClientParams{ @@ -263,21 +253,21 @@ func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { return pool, nil } -func initExportService(ctx context.Context, cfg config.MassExport, redisCfg *config.Redis, client seqdb.Client) (massexport.Service, error) { - sessionStore, err := sessionstore.NewRedisSessionStore(ctx, redisCfg, cfg.SessionStore.ExportLifetime) - if err != nil { - return nil, fmt.Errorf("init session store: %w", err) - } - logger.Info("session store initialized") - - fileStore, err := filestore.NewS3(ctx, cfg.FileStore.S3) - if err != nil { - return nil, fmt.Errorf("init file store: %w", err) - } - logger.Info("file store initialized") - - return massexport.NewService(ctx, cfg, sessionStore, fileStore, client) -} +// func initExportService(ctx context.Context, cfg config.MassExport, redisCfg *config.Redis, client seqdb.Client) (massexport.Service, error) { +// sessionStore, err := sessionstore.NewRedisSessionStore(ctx, redisCfg, cfg.SessionStore.ExportLifetime) +// if err != nil { +// return nil, fmt.Errorf("init session store: %w", err) +// } +// logger.Info("session store initialized") +// +// fileStore, err := filestore.NewS3(ctx, cfg.FileStore.S3) +// if err != nil { +// return nil, fmt.Errorf("init file store: %w", err) +// } +// logger.Info("file store initialized") +// +// return massexport.NewService(ctx, cfg, sessionStore, fileStore, client) +// } func initClickHouseClients(ctx context.Context, cfg *config.Config) (map[string]driver.Conn, error) { clients := make(map[string]driver.Conn, len(cfg.Clients.ClickHouse)) diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 10edcff..9a26415 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -549,14 +549,14 @@ type SeqAPIOptions struct { } type SeqAPILimits struct { - MaxSearchLimit int `yaml:"max_search_limit"` - MaxSearchTotal int `yaml:"max_search_total"` - MaxSearchOffset int `yaml:"max_search_offset"` - MaxExportLimit int `yaml:"max_export_limit"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` + MaxSearchLimit int32 `yaml:"max_search_limit"` + MaxSearchTotal int64 `yaml:"max_search_total"` + MaxSearchOffset int32 `yaml:"max_search_offset"` + MaxExportLimit int32 `yaml:"max_export_limit"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` } type SeqAPICaches struct { diff --git a/internal/app/types/async_searches.go b/internal/app/types/async_searches.go index 9618a71..75ebb6e 100644 --- a/internal/app/types/async_searches.go +++ b/internal/app/types/async_searches.go @@ -7,6 +7,7 @@ type SaveAsyncSearchRequest struct { OwnerID int64 ExpiresAt time.Time Meta string + Env string } type AsyncSearchInfo struct { @@ -22,4 +23,19 @@ type GetAsyncSearchesListRequest struct { Limit int32 Offset int32 Owner *string + Env string +} + +type GetAsyncSearchByIdRequest struct { + SearchID string + Env string +} + +type DeleteAsyncSearchRequest struct { + SearchID string + Env string +} + +type DeleteExpiredAsyncSearchesRequest struct { + Env string } diff --git a/internal/pkg/repository/async_searches_repo.go b/internal/pkg/repository/async_searches_repo.go index 8594134..4313c74 100644 --- a/internal/pkg/repository/async_searches_repo.go +++ b/internal/pkg/repository/async_searches_repo.go @@ -20,12 +20,9 @@ func newAsyncSearchesRepository(pool *pool) *asyncSearchesRepository { return &asyncSearchesRepository{pool} } -func (r *asyncSearchesRepository) SaveAsyncSearch( - ctx context.Context, - req types.SaveAsyncSearchRequest, -) error { - query, args := "INSERT INTO async_searches (search_id,owner_id,meta,expires_at) VALUES ($1,$2,$3,$4)", - []any{req.SearchID, req.OwnerID, req.Meta, req.ExpiresAt} +func (r *asyncSearchesRepository) SaveAsyncSearch(ctx context.Context, req types.SaveAsyncSearchRequest) error { + query, args := "INSERT INTO async_searches (search_id,owner_id,meta,expires_at,env) VALUES ($1,$2,$3,$4,$5)", + []any{req.SearchID, req.OwnerID, req.Meta, req.ExpiresAt, req.Env} metricLabels := []string{"async_searches", "INSERT"} _, err := r.exec(ctx, metricLabels, query, args...) @@ -37,19 +34,16 @@ func (r *asyncSearchesRepository) SaveAsyncSearch( return nil } -func (r *asyncSearchesRepository) GetAsyncSearchById( - ctx context.Context, - searchID string, -) (types.AsyncSearchInfo, error) { +func (r *asyncSearchesRepository) GetAsyncSearchById(ctx context.Context, req types.GetAsyncSearchByIdRequest) (types.AsyncSearchInfo, error) { as := types.AsyncSearchInfo{} query, args := ` SELECT s.search_id, s.owner_id, p.user_name, s.meta, s.created_at, s.expires_at FROM async_searches AS s JOIN user_profiles AS p ON p.id = s.owner_id - WHERE s.search_id = $1 + WHERE s.search_id = $1 AND s.env = $2 `, - []any{searchID} + []any{req.SearchID, req.Env} metricLabels := []string{"async_searches", "SELECT"} err := r.queryRow(ctx, metricLabels, query, args...).Scan( @@ -74,13 +68,11 @@ func (r *asyncSearchesRepository) GetAsyncSearchById( return as, nil } -func (r *asyncSearchesRepository) GetAsyncSearchesList( - ctx context.Context, - req types.GetAsyncSearchesListRequest, -) ([]types.AsyncSearchInfo, error) { +func (r *asyncSearchesRepository) GetAsyncSearchesList(ctx context.Context, req types.GetAsyncSearchesListRequest) ([]types.AsyncSearchInfo, error) { qb := sqlb.Select("s.search_id", "s.owner_id", "p.user_name", "s.created_at", "s.expires_at"). From("async_searches AS s"). Join("user_profiles AS p ON p.id = s.owner_id"). + Where(sq.Eq{"s.env": req.Env}). Where(sq.GtOrEq{"s.expires_at": "now()"}). OrderBy("s.created_at DESC") @@ -119,12 +111,9 @@ func (r *asyncSearchesRepository) GetAsyncSearchesList( return asyncSearches, nil } -func (r *asyncSearchesRepository) DeleteAsyncSearch( - ctx context.Context, - searchID string, -) error { - query, args := "DELETE FROM async_searches WHERE search_id = $1", - []any{searchID} +func (r *asyncSearchesRepository) DeleteAsyncSearch(ctx context.Context, req types.DeleteAsyncSearchRequest) error { + query, args := "DELETE FROM async_searches WHERE search_id = $1 AND env = $2", + []any{req.SearchID, req.Env} metricLabelsDelete := []string{"async_searches", "DELETE"} if _, err := r.exec(ctx, metricLabelsDelete, query, args...); err != nil { @@ -135,11 +124,12 @@ func (r *asyncSearchesRepository) DeleteAsyncSearch( return nil } -func (r *asyncSearchesRepository) DeleteExpiredAsyncSearches(ctx context.Context) error { - query := "DELETE FROM async_searches WHERE expires_at < now()" +func (r *asyncSearchesRepository) DeleteExpiredAsyncSearches(ctx context.Context, req types.DeleteExpiredAsyncSearchesRequest) error { + query, args := "DELETE FROM async_searches WHERE env = $1 expires_at < now()", + []any{req.Env} metricLabelsDelete := []string{"async_searches", "DELETE"} - if _, err := r.exec(ctx, metricLabelsDelete, query); err != nil { + if _, err := r.exec(ctx, metricLabelsDelete, query, args...); err != nil { incErrorMetric(err, metricLabelsDelete) return fmt.Errorf("failed to delete expired async searches: %w", err) } diff --git a/internal/pkg/repository/repository.go b/internal/pkg/repository/repository.go index 6087d20..d425960 100644 --- a/internal/pkg/repository/repository.go +++ b/internal/pkg/repository/repository.go @@ -33,9 +33,9 @@ type ( AsyncSearches interface { SaveAsyncSearch(context.Context, types.SaveAsyncSearchRequest) error - GetAsyncSearchById(context.Context, string) (types.AsyncSearchInfo, error) - DeleteAsyncSearch(context.Context, string) error - DeleteExpiredAsyncSearches(context.Context) error + GetAsyncSearchById(context.Context, types.GetAsyncSearchByIdRequest) (types.AsyncSearchInfo, error) + DeleteAsyncSearch(context.Context, types.DeleteAsyncSearchRequest) error + DeleteExpiredAsyncSearches(context.Context, types.DeleteExpiredAsyncSearchesRequest) error GetAsyncSearchesList(context.Context, types.GetAsyncSearchesListRequest) ([]types.AsyncSearchInfo, error) } diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index ea37a1e..b185736 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -41,14 +41,16 @@ type service struct { repo repository.AsyncSearches seqDB seqdb.Client - cfg config.AsyncSearch + cfg *config.AsyncSearchOptions + env string } -func New(ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, cfg config.AsyncSearch) Service { +func New(ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, cfg *config.AsyncSearchOptions, env string) Service { s := &service{ repo: repo, seqDB: seqDB, cfg: cfg, + env: env, } go s.deleteExpiredAsyncSearches(ctx) @@ -77,6 +79,7 @@ func (s *service) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSe OwnerID: ownerID, ExpiresAt: time.Now().Add(req.Retention.AsDuration()), Meta: req.Meta, + Env: s.env, }) }, backoff.NewExponentialBackOff( backoff.WithInitialInterval(backoffInitialInterval), @@ -95,7 +98,10 @@ func (s *service) DeleteAsyncSearch(ctx context.Context, req *seqapi.DeleteAsync return nil, err } - searchInfo, err := s.repo.GetAsyncSearchById(ctx, req.SearchId) + searchInfo, err := s.repo.GetAsyncSearchById(ctx, types.GetAsyncSearchByIdRequest{ + SearchID: req.SearchId, + Env: s.env, + }) if err != nil { return nil, fmt.Errorf("failed to get async search by id: %w", err) } @@ -111,7 +117,10 @@ func (s *service) DeleteAsyncSearch(ctx context.Context, req *seqapi.DeleteAsync // we can retry multiple times after the async search is deleted from seq-db err = backoff.Retry(func() error { - return s.repo.DeleteAsyncSearch(ctx, req.SearchId) + return s.repo.DeleteAsyncSearch(ctx, types.DeleteAsyncSearchRequest{ + SearchID: req.SearchId, + Env: s.env, + }) }, backoff.NewExponentialBackOff( backoff.WithInitialInterval(backoffInitialInterval), backoff.WithMaxElapsedTime(maxBackoffElapsedTime), @@ -129,7 +138,10 @@ func (s *service) CancelAsyncSearch(ctx context.Context, req *seqapi.CancelAsync return nil, err } - searchInfo, err := s.repo.GetAsyncSearchById(ctx, req.SearchId) + searchInfo, err := s.repo.GetAsyncSearchById(ctx, types.GetAsyncSearchByIdRequest{ + SearchID: req.SearchId, + Env: s.env, + }) if err != nil { return nil, fmt.Errorf("failed to get async search by id: %w", err) } @@ -147,7 +159,10 @@ func (s *service) CancelAsyncSearch(ctx context.Context, req *seqapi.CancelAsync } func (s *service) FetchAsyncSearchResult(ctx context.Context, req *seqapi.FetchAsyncSearchResultRequest) (*seqapi.FetchAsyncSearchResultResponse, error) { - searchInfo, err := s.repo.GetAsyncSearchById(ctx, req.SearchId) + searchInfo, err := s.repo.GetAsyncSearchById(ctx, types.GetAsyncSearchByIdRequest{ + SearchID: req.SearchId, + Env: s.env, + }) if err != nil { return nil, fmt.Errorf("failed to get async search by id: %w", err) } @@ -172,6 +187,7 @@ func (s *service) ExportAsyncSearch(ctx context.Context, req *seqapi.ExportAsync func (s *service) GetAsyncSearchesList(ctx context.Context, req *seqapi.GetAsyncSearchesListRequest) (*seqapi.GetAsyncSearchesListResponse, error) { searches, err := s.repo.GetAsyncSearchesList(ctx, types.GetAsyncSearchesListRequest{ Owner: req.OwnerName, + Env: s.env, }) if err != nil { return nil, fmt.Errorf("failed to get async searches list from db: %w", err) @@ -219,7 +235,9 @@ func (s *service) deleteExpiredAsyncSearches(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - err := s.repo.DeleteExpiredAsyncSearches(ctx) + err := s.repo.DeleteExpiredAsyncSearches(ctx, types.DeleteExpiredAsyncSearchesRequest{ + Env: s.env, + }) if err != nil { logger.Error("DeleteExpiredAsyncSearches error", zap.Error(err)) } diff --git a/migration/12_alter_table_async_searches_add_column_env.sql b/migration/12_alter_table_async_searches_add_column_env.sql new file mode 100644 index 0000000..52ba0f2 --- /dev/null +++ b/migration/12_alter_table_async_searches_add_column_env.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE IF EXISTS async_searches ADD COLUMN IF NOT EXISTS env text NOT NULL DEFAULT ''; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE IF EXISTS async_searches DROP COLUMN IF EXISTS env; +-- +goose StatementEnd From 37640c109e48eb890f865f7a400a4e243fbab29b Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Thu, 15 Oct 2026 23:44:51 +0300 Subject: [PATCH 23/29] update errorgroups/admin - admin without migration/repo --- cmd/seq-ui/main.go | 30 +- internal/api/admin/v1/admin.go | 6 +- internal/api/admin/v1/grpc/api.go | 15 +- internal/api/admin/v1/grpc/roles.go | 21 +- internal/api/admin/v1/grpc/roles_test.go | 3 +- internal/api/admin/v1/http/api.go | 15 +- internal/api/admin/v1/http/roles.go | 28 +- internal/api/admin/v1/http/roles_test.go | 3 +- internal/api/errorgroups/v1/errorgroups.go | 7 +- internal/api/errorgroups/v1/grpc/api.go | 16 +- .../errorgroups/v1/grpc/diff_by_releases.go | 10 +- .../v1/grpc/diff_by_releases_test.go | 3 +- .../api/errorgroups/v1/grpc/get_details.go | 10 +- .../errorgroups/v1/grpc/get_details_test.go | 3 +- .../api/errorgroups/v1/grpc/get_groups.go | 12 +- .../errorgroups/v1/grpc/get_groups_test.go | 3 +- internal/api/errorgroups/v1/grpc/get_hist.go | 10 +- .../api/errorgroups/v1/grpc/get_hist_test.go | 3 +- .../api/errorgroups/v1/grpc/get_releases.go | 10 +- .../errorgroups/v1/grpc/get_releases_test.go | 3 +- .../v1/grpc/get_service_filters.go | 10 +- .../v1/grpc/get_service_filters_test.go | 3 +- .../api/errorgroups/v1/grpc/get_services.go | 10 +- .../errorgroups/v1/grpc/get_services_test.go | 3 +- .../api/errorgroups/v1/grpc/get_top_groups.go | 10 +- .../v1/grpc/get_top_groups_test.go | 3 +- internal/api/errorgroups/v1/http/api.go | 15 +- .../errorgroups/v1/http/diff_by_releases.go | 11 +- .../v1/http/diff_by_releases_test.go | 3 +- .../api/errorgroups/v1/http/get_details.go | 11 +- .../errorgroups/v1/http/get_details_test.go | 3 +- .../api/errorgroups/v1/http/get_groups.go | 13 +- .../errorgroups/v1/http/get_groups_test.go | 3 +- internal/api/errorgroups/v1/http/get_hist.go | 11 +- .../api/errorgroups/v1/http/get_hist_test.go | 3 +- .../api/errorgroups/v1/http/get_releases.go | 11 +- .../errorgroups/v1/http/get_releases_test.go | 3 +- .../v1/http/get_service_filters.go | 11 +- .../v1/http/get_service_filters_test.go | 3 +- .../api/errorgroups/v1/http/get_services.go | 11 +- .../errorgroups/v1/http/get_services_test.go | 3 +- .../api/errorgroups/v1/http/get_top_groups.go | 11 +- .../v1/http/get_top_groups_test.go | 3 +- internal/api/seqapi/v1/grpc/api.go | 3 +- .../api/seqapi/v1/http/cancel_async_search.go | 1 + .../api/seqapi/v1/http/delete_async_search.go | 1 + internal/api/seqapi/v1/http/events_test.go | 2 +- .../v1/http/fetch_async_search_result.go | 1 + internal/api/seqapi/v1/http/fields_test.go | 2 +- .../seqapi/v1/http/get_async_searches_list.go | 1 + .../api/seqapi/v1/http/start_async_search.go | 1 + internal/api/seqapi/v1/seqapi.go | 35 +- internal/app/config/migrate/v1_v2.go | 370 +++++++++--------- internal/app/types/admin.go | 5 + internal/pkg/repository/mock/repository.go | 12 +- internal/pkg/service/admin/cache.go | 20 +- internal/pkg/service/admin/permissions.go | 5 +- .../pkg/service/admin/permissions_test.go | 10 +- internal/pkg/service/admin/roles.go | 7 +- internal/pkg/service/admin/roles_test.go | 27 +- internal/pkg/service/admin/service.go | 33 +- .../pkg/service/async_searches/service.go | 27 ++ internal/pkg/service/errorgroups/service.go | 20 + swagger/swagger.json | 143 ++++++- 64 files changed, 694 insertions(+), 411 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 8f9d0b1..4cdf3b5 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -20,6 +20,7 @@ import ( "github.com/ozontech/seq-ui/internal/api" admin_v1 "github.com/ozontech/seq-ui/internal/api/admin/v1" dashboards_v1 "github.com/ozontech/seq-ui/internal/api/dashboards/v1" + errorgroups_v1 "github.com/ozontech/seq-ui/internal/api/errorgroups/v1" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" "github.com/ozontech/seq-ui/internal/app/config/loader" @@ -28,7 +29,10 @@ import ( "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/repository" + "github.com/ozontech/seq-ui/internal/pkg/service/admin" + asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" "github.com/ozontech/seq-ui/internal/pkg/service/dashboards" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" "github.com/ozontech/seq-ui/internal/pkg/service/profiles" "github.com/ozontech/seq-ui/internal/pkg/service/userprofile" "github.com/ozontech/seq-ui/logger" @@ -135,9 +139,9 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { var ( adminV1 *admin_v1.Admin - asyncSearchesRepo repository.AsyncSearches userProfileV1 *userprofile_v1.UserProfile dashboardsV1 *dashboards_v1.Dashboards + asyncSearchesRepo repository.AsyncSearches ) if db != nil { repo := repository.New(db, cfg.DB.RequestTimeout) @@ -155,24 +159,26 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { asyncSearchesRepo = repo.AsyncSearches } - // if cfg.Handlers.Admin != nil { - // logger.Info("initializing admin") - // adminSvc := admin.New(repo.Admin, repo.UserProfiles, adminCache, cfg.Handlers.Admin) - // adminV1 = admin_v1.New(adminSvc) - // } + if cfg.Handlers.Admin != nil { + logger.Info("initializing admin") + adminSvc := admin.NewByEnv(cfg.Handlers.Admin, repo.Admin, repo.UserProfiles) + adminV1 = admin_v1.New(adminSvc) + } } var seqApiV1 *seqapi_v1.SeqAPI if cfg.Handlers.SeqAPI != nil { logger.Info("initializing seqapi") - seqApiV1 = seqapi_v1.New(ctx, cfg.Handlers.SeqAPI, seqDBClients, cfg.Handlers.AsyncSearch, asyncSearchesRepo) + asyncSearchesSvc := asyncsearches.NewByEnv(ctx, cfg.Handlers.AsyncSearch, asyncSearchesRepo, seqDBClients) + seqApiV1 = seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, asyncSearchesSvc) } - // var errorGroupsV1 *errorgroups_v1.ErrorGroups - // if cfg.Handlers.ErrorGroups != nil { - // logger.Info("initializing error groups") - // errorGroupsV1 = errorgroups_v1.New(cfg.Handlers.ErrorGroups, &cfg.Clients, chClients) - // } + var errorGroupsV1 *errorgroups_v1.ErrorGroups + if cfg.Handlers.ErrorGroups != nil { + logger.Info("initializing error groups") + errorGroupsSvc := errorgroups.NewByEnv(cfg.Handlers.ErrorGroups, &cfg.Clients, chClients) + errorGroupsV1 = errorgroups_v1.New(errorGroupsSvc) + } return api.NewRegistrar(adminV1, seqApiV1, userProfileV1, dashboardsV1, massExportV1, errorGroupsV1) } diff --git a/internal/api/admin/v1/admin.go b/internal/api/admin/v1/admin.go index 9bcee27..ac8c549 100644 --- a/internal/api/admin/v1/admin.go +++ b/internal/api/admin/v1/admin.go @@ -13,10 +13,10 @@ type Admin struct { httpAPI *http_api.API } -func New(svc admin.Service) *Admin { +func New(services map[string]admin.Service) *Admin { return &Admin{ - grpcAPI: grpc_api.New(svc), - httpAPI: http_api.New(svc), + grpcAPI: grpc_api.New(services), + httpAPI: http_api.New(services), } } diff --git a/internal/api/admin/v1/grpc/api.go b/internal/api/admin/v1/grpc/api.go index 1368bc4..12072f2 100644 --- a/internal/api/admin/v1/grpc/api.go +++ b/internal/api/admin/v1/grpc/api.go @@ -9,14 +9,21 @@ import ( type API struct { api.UnimplementedAdminServiceServer - service admin.Service + services map[string]admin.Service availablePermissions []*api.PermissionGroup } -func New(svc admin.Service) *API { +func New(services map[string]admin.Service) *API { + // available permissions are the same for all envs + var availablePermissions []*api.PermissionGroup + for _, svc := range services { + availablePermissions = availablePermissionsToProto(svc.GetAvailablePermissions()) + break + } + return &API{ - service: svc, - availablePermissions: availablePermissionsToProto(svc.GetAvailablePermissions()), + services: services, + availablePermissions: availablePermissions, } } diff --git a/internal/api/admin/v1/grpc/roles.go b/internal/api/admin/v1/grpc/roles.go index c41e695..2e9e397 100644 --- a/internal/api/admin/v1/grpc/roles.go +++ b/internal/api/admin/v1/grpc/roles.go @@ -34,7 +34,8 @@ func (a *API) CreateRole(ctx context.Context, req *admin.CreateRoleRequest) (*ad Permissions: permissionGroupsToStrings(req.Permissions), } - roleID, err := a.service.CreateRole(ctx, request) + svc := a.services[types.GetEnvKey(ctx)] + roleID, err := svc.CreateRole(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } @@ -59,7 +60,8 @@ func (a *API) AddUsersToRole(ctx context.Context, req *admin.AddUsersToRoleReque }, ) - if err := a.service.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ RoleID: req.RoleId, Usernames: req.Usernames, }); err != nil { @@ -84,7 +86,8 @@ func (a *API) DeleteUsersFromRole(ctx context.Context, req *admin.DeleteUsersFro }, ) - if err := a.service.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ RoleID: req.RoleId, Usernames: req.Usernames, }); err != nil { @@ -95,7 +98,8 @@ func (a *API) DeleteUsersFromRole(ctx context.Context, req *admin.DeleteUsersFro } func (a *API) GetRoles(ctx context.Context, _ *admin.GetRolesRequest) (*admin.GetRolesResponse, error) { - resp, err := a.service.GetRoles(ctx) + svc := a.services[types.GetEnvKey(ctx)] + resp, err := svc.GetRoles(ctx) if err != nil { return nil, grpcutil.ProcessError(err) } @@ -117,7 +121,8 @@ func (a *API) GetRole(ctx context.Context, req *admin.GetRoleRequest) (*admin.Ge }, ) - roleInfo, err := a.service.GetRole(ctx, types.GetRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + roleInfo, err := svc.GetRole(ctx, types.GetRoleRequest{ RoleID: req.Id, }) if err != nil { @@ -151,7 +156,8 @@ func (a *API) UpdateRole(ctx context.Context, req *admin.UpdateRoleRequest) (*ad } span.SetAttributes(spanAttributes...) - if err := a.service.UpdateRole(ctx, types.UpdateRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.UpdateRole(ctx, types.UpdateRoleRequest{ RoleID: req.Id, Name: req.Name, Permissions: permissionGroupsToStrings(req.Permissions), @@ -180,7 +186,8 @@ func (a *API) DeleteRole(ctx context.Context, req *admin.DeleteRoleRequest) (*ad } span.SetAttributes(spanAttributes...) - if err := a.service.DeleteRole(ctx, types.DeleteRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.DeleteRole(ctx, types.DeleteRoleRequest{ RoleID: req.Id, ReplacementRoleID: req.ReplacementRoleId, }); err != nil { diff --git a/internal/api/admin/v1/grpc/roles_test.go b/internal/api/admin/v1/grpc/roles_test.go index d10785e..5a45f5d 100644 --- a/internal/api/admin/v1/grpc/roles_test.go +++ b/internal/api/admin/v1/grpc/roles_test.go @@ -11,6 +11,7 @@ import ( "google.golang.org/grpc/status" "github.com/ozontech/seq-ui/internal/app/types" + admin_service "github.com/ozontech/seq-ui/internal/pkg/service/admin" mock "github.com/ozontech/seq-ui/internal/pkg/service/admin/mock" "github.com/ozontech/seq-ui/pkg/admin/v1" ) @@ -33,7 +34,7 @@ func setupTestAPI(t *testing.T) (*API, *mock.MockService) { }). Times(1) - return New(mockedSvc), mockedSvc + return New(map[string]admin_service.Service{"": mockedSvc}), mockedSvc } func TestCreateRole(t *testing.T) { diff --git a/internal/api/admin/v1/http/api.go b/internal/api/admin/v1/http/api.go index b2e38a0..823a2f5 100644 --- a/internal/api/admin/v1/http/api.go +++ b/internal/api/admin/v1/http/api.go @@ -8,14 +8,21 @@ import ( ) type API struct { - service adminservice.Service + services map[string]adminservice.Service availablePermissions []permissionGroup } -func New(svc adminservice.Service) *API { +func New(services map[string]adminservice.Service) *API { + // available permissions are the same for all envs + var availablePermissions []permissionGroup + for _, svc := range services { + availablePermissions = parseAvailablePermissions(svc.GetAvailablePermissions()) + break + } + return &API{ - service: svc, - availablePermissions: parseAvailablePermissions(svc.GetAvailablePermissions()), + services: services, + availablePermissions: availablePermissions, } } diff --git a/internal/api/admin/v1/http/roles.go b/internal/api/admin/v1/http/roles.go index 04ab891..467b976 100644 --- a/internal/api/admin/v1/http/roles.go +++ b/internal/api/admin/v1/http/roles.go @@ -24,6 +24,7 @@ const permSep = ":" // @Router /admin/v1/roles [post] // @ID admin_v1_create_role // @Tags admin_v1 +// @Param env query string false "Environment" // @Param body body createRoleRequest true "Request body" // @Success 200 {object} createRoleResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -51,7 +52,8 @@ func (a *API) serveCreateRole(w http.ResponseWriter, r *http.Request) { }, ) - roleID, err := a.service.CreateRole(ctx, types.CreateRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + roleID, err := svc.CreateRole(ctx, types.CreateRoleRequest{ Name: httpReq.Name, Permissions: parsePermissionGroupsToStrings(httpReq.Permissions), }) @@ -69,6 +71,7 @@ func (a *API) serveCreateRole(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_add_users_to_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Param body body addUsersToRoleRequest true "Request body" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -102,7 +105,8 @@ func (a *API) serveAddUsersToRole(w http.ResponseWriter, r *http.Request) { }, ) - if err := a.service.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ RoleID: roleID, Usernames: httpReq.Usernames, }); err != nil { @@ -119,6 +123,7 @@ func (a *API) serveAddUsersToRole(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_delete_users_from_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Param body body deleteUsersFromRoleRequest true "Request body" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -152,7 +157,8 @@ func (a *API) serveDeleteUsersFromRole(w http.ResponseWriter, r *http.Request) { }, ) - if err := a.service.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ RoleID: roleID, Usernames: httpReq.Usernames, }); err != nil { @@ -168,6 +174,7 @@ func (a *API) serveDeleteUsersFromRole(w http.ResponseWriter, r *http.Request) { // @Router /admin/v1/roles [get] // @ID admin_v1_get_roles // @Tags admin_v1 +// @Param env query string false "Environment" // @Success 200 {object} getRolesResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer @@ -177,7 +184,8 @@ func (a *API) serveGetRoles(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - resp, err := a.service.GetRoles(ctx) + svc := a.services[types.GetEnvKey(ctx)] + resp, err := svc.GetRoles(ctx) if err != nil { httputil.ProcessError(wr, err) return @@ -195,6 +203,7 @@ func (a *API) serveGetRoles(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_get_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Success 200 {object} getRoleResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer @@ -217,7 +226,8 @@ func (a *API) serveGetRole(w http.ResponseWriter, r *http.Request) { }, ) - roleInfo, err := a.service.GetRole(ctx, types.GetRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + roleInfo, err := svc.GetRole(ctx, types.GetRoleRequest{ RoleID: roleID, }) if err != nil { @@ -236,6 +246,7 @@ func (a *API) serveGetRole(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_update_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Param body body updateRoleRequest true "Request body" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -276,7 +287,8 @@ func (a *API) serveUpdateRole(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(spanAttributes...) - if err := a.service.UpdateRole(ctx, types.UpdateRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.UpdateRole(ctx, types.UpdateRoleRequest{ RoleID: roleID, Name: httpReq.Name, Permissions: parsePermissionGroupsToStrings(httpReq.Permissions), @@ -294,6 +306,7 @@ func (a *API) serveUpdateRole(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_delete_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Param body body deleteRoleRequest false "Request body" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -330,7 +343,8 @@ func (a *API) serveDeleteRole(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(spanAttributes...) - if err := a.service.DeleteRole(ctx, types.DeleteRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.DeleteRole(ctx, types.DeleteRoleRequest{ RoleID: roleID, ReplacementRoleID: httpReq.ReplacementRoleID, }); err != nil { diff --git a/internal/api/admin/v1/http/roles_test.go b/internal/api/admin/v1/http/roles_test.go index 9876166..c4281f8 100644 --- a/internal/api/admin/v1/http/roles_test.go +++ b/internal/api/admin/v1/http/roles_test.go @@ -11,6 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + admin_service "github.com/ozontech/seq-ui/internal/pkg/service/admin" mock "github.com/ozontech/seq-ui/internal/pkg/service/admin/mock" ) @@ -32,7 +33,7 @@ func setupTestAPI(t *testing.T) (*API, *mock.MockService) { }). Times(1) - return New(mockedSvc), mockedSvc + return New(map[string]admin_service.Service{"": mockedSvc}), mockedSvc } func withRoleID(h http.HandlerFunc, id string) http.HandlerFunc { diff --git a/internal/api/errorgroups/v1/errorgroups.go b/internal/api/errorgroups/v1/errorgroups.go index 91af47a..97e0d7a 100644 --- a/internal/api/errorgroups/v1/errorgroups.go +++ b/internal/api/errorgroups/v1/errorgroups.go @@ -1,7 +1,6 @@ package errorgroups_v1 import ( - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "github.com/go-chi/chi/v5" grpc_api "github.com/ozontech/seq-ui/internal/api/errorgroups/v1/grpc" @@ -14,10 +13,10 @@ type ErrorGroups struct { httpAPI *http_api.API } -func New(svc errorgroups.Service, chClients map[string]driver.Conn) *ErrorGroups { +func New(services map[string]errorgroups.Service) *ErrorGroups { return &ErrorGroups{ - grpcAPI: grpc_api.New(svc), - httpAPI: http_api.New(svc), + grpcAPI: grpc_api.New(services), + httpAPI: http_api.New(services), } } diff --git a/internal/api/errorgroups/v1/grpc/api.go b/internal/api/errorgroups/v1/grpc/api.go index 62071e5..fb807ac 100644 --- a/internal/api/errorgroups/v1/grpc/api.go +++ b/internal/api/errorgroups/v1/grpc/api.go @@ -1,6 +1,8 @@ package grpc import ( + "context" + "google.golang.org/protobuf/types/known/durationpb" "github.com/ozontech/seq-ui/internal/app/types" @@ -11,13 +13,21 @@ import ( type API struct { api.UnimplementedErrorGroupsServiceServer - service errorgroups.Service + services map[string]errorgroups.Service } -func New(svc errorgroups.Service) *API { +func New(services map[string]errorgroups.Service) *API { return &API{ - service: svc, + services: services, + } +} + +func (a *API) serviceByEnv(ctx context.Context, reqEnv *string) (errorgroups.Service, *string) { + env := types.GetEnvKey(ctx) + if (reqEnv == nil || *reqEnv == "") && env != "" { + reqEnv = &env } + return a.services[env], reqEnv } type trReq interface { diff --git a/internal/api/errorgroups/v1/grpc/diff_by_releases.go b/internal/api/errorgroups/v1/grpc/diff_by_releases.go index f6e7fdd..7b68c33 100644 --- a/internal/api/errorgroups/v1/grpc/diff_by_releases.go +++ b/internal/api/errorgroups/v1/grpc/diff_by_releases.go @@ -16,6 +16,8 @@ func (a *API) DiffByReleases(ctx context.Context, req *errorgroups.DiffByRelease ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_diff_by_releases") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, {Key: "releases", Value: attribute.StringSliceValue(req.Releases)}, @@ -24,18 +26,18 @@ func (a *API) DiffByReleases(ctx context.Context, req *errorgroups.DiffByRelease {Key: "order", Value: attribute.StringValue(req.Order.String())}, {Key: "with_total", Value: attribute.BoolValue(req.WithTotal)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Source != nil { attributes = append(attributes, attribute.KeyValue{Key: "source", Value: attribute.StringValue(*req.Source)}) } span.SetAttributes(attributes...) - groups, total, err := a.service.DiffByReleases(ctx, types.DiffByReleasesRequest{ + groups, total, err := svc.DiffByReleases(ctx, types.DiffByReleasesRequest{ Service: req.Service, Releases: req.Releases, - Env: req.Env, + Env: reqEnv, Source: req.Source, Limit: req.Limit, Offset: req.Offset, diff --git a/internal/api/errorgroups/v1/grpc/diff_by_releases_test.go b/internal/api/errorgroups/v1/grpc/diff_by_releases_test.go index d0cef71..ddd6948 100644 --- a/internal/api/errorgroups/v1/grpc/diff_by_releases_test.go +++ b/internal/api/errorgroups/v1/grpc/diff_by_releases_test.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -145,7 +146,7 @@ func TestDiffByReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_details.go b/internal/api/errorgroups/v1/grpc/get_details.go index a923fa8..091d50c 100644 --- a/internal/api/errorgroups/v1/grpc/get_details.go +++ b/internal/api/errorgroups/v1/grpc/get_details.go @@ -20,14 +20,16 @@ func (a *API) GetDetails(ctx context.Context, req *errorgroups.GetDetailsRequest ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_details") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "group_hash", Value: attribute.StringValue(strconv.FormatUint(req.GroupHash, 10))}, } if req.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*req.Service)}) } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*req.Release)}) @@ -44,7 +46,7 @@ func (a *API) GetDetails(ctx context.Context, req *errorgroups.GetDetailsRequest request := types.GetErrorGroupDetailsRequest{ Service: req.Service, GroupHash: req.GroupHash, - Env: req.Env, + Env: reqEnv, Source: req.Source, Release: req.Release, } @@ -55,7 +57,7 @@ func (a *API) GetDetails(ctx context.Context, req *errorgroups.GetDetailsRequest } } - details, err := a.service.GetDetails(ctx, request) + details, err := svc.GetDetails(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_details_test.go b/internal/api/errorgroups/v1/grpc/get_details_test.go index 2a0d007..21c7191 100644 --- a/internal/api/errorgroups/v1/grpc/get_details_test.go +++ b/internal/api/errorgroups/v1/grpc/get_details_test.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -261,7 +262,7 @@ func TestGetDetails(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_groups.go b/internal/api/errorgroups/v1/grpc/get_groups.go index 0a589ea..52baad5 100644 --- a/internal/api/errorgroups/v1/grpc/get_groups.go +++ b/internal/api/errorgroups/v1/grpc/get_groups.go @@ -17,6 +17,8 @@ func (a *API) GetGroups(ctx context.Context, req *errorgroups.GetGroupsRequest) ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_groups") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, {Key: "limit", Value: attribute.IntValue(int(req.Limit))}, @@ -24,8 +26,8 @@ func (a *API) GetGroups(ctx context.Context, req *errorgroups.GetGroupsRequest) {Key: "order", Value: attribute.StringValue(string(req.Order))}, {Key: "with_total", Value: attribute.BoolValue(req.WithTotal)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*req.Release)}) @@ -48,7 +50,7 @@ func (a *API) GetGroups(ctx context.Context, req *errorgroups.GetGroupsRequest) request := types.GetErrorGroupsRequest{ Service: req.Service, - Env: req.Env, + Env: reqEnv, Source: req.Source, Release: req.Release, TimeRange: parseTimeRange(req), @@ -70,9 +72,9 @@ func (a *API) GetGroups(ctx context.Context, req *errorgroups.GetGroupsRequest) err error ) if req.Filter != nil && req.Filter.IsNew { - groups, total, err = a.service.GetNewErrorGroups(ctx, request) + groups, total, err = svc.GetNewErrorGroups(ctx, request) } else { - groups, total, err = a.service.GetErrorGroups(ctx, request) + groups, total, err = svc.GetErrorGroups(ctx, request) } if err != nil { diff --git a/internal/api/errorgroups/v1/grpc/get_groups_test.go b/internal/api/errorgroups/v1/grpc/get_groups_test.go index 2ec2993..9083eb5 100644 --- a/internal/api/errorgroups/v1/grpc/get_groups_test.go +++ b/internal/api/errorgroups/v1/grpc/get_groups_test.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -445,7 +446,7 @@ func TestGetGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { if tt.req.Filter != nil && tt.req.Filter.IsNew { diff --git a/internal/api/errorgroups/v1/grpc/get_hist.go b/internal/api/errorgroups/v1/grpc/get_hist.go index 6672e12..eaf1649 100644 --- a/internal/api/errorgroups/v1/grpc/get_hist.go +++ b/internal/api/errorgroups/v1/grpc/get_hist.go @@ -18,6 +18,8 @@ func (a *API) GetHist(ctx context.Context, req *errorgroups.GetHistRequest) (*er ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_hist") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{} if req.GroupHash != nil { attributes = append(attributes, attribute.KeyValue{ @@ -27,8 +29,8 @@ func (a *API) GetHist(ctx context.Context, req *errorgroups.GetHistRequest) (*er if req.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*req.Service)}) } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*req.Release)}) @@ -52,7 +54,7 @@ func (a *API) GetHist(ctx context.Context, req *errorgroups.GetHistRequest) (*er request := types.GetErrorHistRequest{ Service: req.Service, GroupHash: req.GroupHash, - Env: req.Env, + Env: reqEnv, Source: req.Source, Release: req.Release, TimeRange: parseTimeRange(req), @@ -64,7 +66,7 @@ func (a *API) GetHist(ctx context.Context, req *errorgroups.GetHistRequest) (*er } } - hist, err := a.service.GetHist(ctx, request) + hist, err := svc.GetHist(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_hist_test.go b/internal/api/errorgroups/v1/grpc/get_hist_test.go index db06525..04c3c72 100644 --- a/internal/api/errorgroups/v1/grpc/get_hist_test.go +++ b/internal/api/errorgroups/v1/grpc/get_hist_test.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -231,7 +232,7 @@ func TestGetHist(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_releases.go b/internal/api/errorgroups/v1/grpc/get_releases.go index 1a5ff8b..7f27655 100644 --- a/internal/api/errorgroups/v1/grpc/get_releases.go +++ b/internal/api/errorgroups/v1/grpc/get_releases.go @@ -15,19 +15,21 @@ func (a *API) GetReleases(ctx context.Context, req *errorgroups.GetReleasesReque ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_releases") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } span.SetAttributes(attributes...) request := types.GetReleasesRequest{ Service: req.Service, - Env: req.Env, + Env: reqEnv, } - releases, err := a.service.GetReleases(ctx, request) + releases, err := svc.GetReleases(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_releases_test.go b/internal/api/errorgroups/v1/grpc/get_releases_test.go index 670bad8..757e993 100644 --- a/internal/api/errorgroups/v1/grpc/get_releases_test.go +++ b/internal/api/errorgroups/v1/grpc/get_releases_test.go @@ -9,6 +9,7 @@ import ( "go.uber.org/mock/gomock" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -76,7 +77,7 @@ func TestGetReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_service_filters.go b/internal/api/errorgroups/v1/grpc/get_service_filters.go index 4b669a7..102a0b2 100644 --- a/internal/api/errorgroups/v1/grpc/get_service_filters.go +++ b/internal/api/errorgroups/v1/grpc/get_service_filters.go @@ -15,19 +15,21 @@ func (a *API) GetServiceFilters(ctx context.Context, req *errorgroups.GetService ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_service_filters") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } span.SetAttributes(attributes...) request := types.GetServiceFiltersRequest{ Service: req.Service, - Env: req.Env, + Env: reqEnv, } - filters, err := a.service.GetServiceFilters(ctx, request) + filters, err := svc.GetServiceFilters(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_service_filters_test.go b/internal/api/errorgroups/v1/grpc/get_service_filters_test.go index d722e1e..73082ab 100644 --- a/internal/api/errorgroups/v1/grpc/get_service_filters_test.go +++ b/internal/api/errorgroups/v1/grpc/get_service_filters_test.go @@ -9,6 +9,7 @@ import ( "go.uber.org/mock/gomock" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -83,7 +84,7 @@ func TestGetServiceFilters(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_services.go b/internal/api/errorgroups/v1/grpc/get_services.go index 69aa0e4..215eb2e 100644 --- a/internal/api/errorgroups/v1/grpc/get_services.go +++ b/internal/api/errorgroups/v1/grpc/get_services.go @@ -15,21 +15,23 @@ func (a *API) GetServices(ctx context.Context, req *errorgroups.GetServicesReque ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_services") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "query", Value: attribute.StringValue(req.Query)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } span.SetAttributes(attributes...) request := types.GetServicesRequest{ Query: req.Query, - Env: req.Env, + Env: reqEnv, Limit: req.Limit, Offset: req.Offset, } - services, err := a.service.GetServices(ctx, request) + services, err := svc.GetServices(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_services_test.go b/internal/api/errorgroups/v1/grpc/get_services_test.go index 2b2b9e5..4ff3ba5 100644 --- a/internal/api/errorgroups/v1/grpc/get_services_test.go +++ b/internal/api/errorgroups/v1/grpc/get_services_test.go @@ -9,6 +9,7 @@ import ( "go.uber.org/mock/gomock" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -81,7 +82,7 @@ func TestGetServices(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_top_groups.go b/internal/api/errorgroups/v1/grpc/get_top_groups.go index 649150f..475122b 100644 --- a/internal/api/errorgroups/v1/grpc/get_top_groups.go +++ b/internal/api/errorgroups/v1/grpc/get_top_groups.go @@ -16,13 +16,15 @@ func (a *API) GetTopGroups(ctx context.Context, req *errorgroups.GetTopGroupsReq ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_top_groups") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "limit", Value: attribute.IntValue(int(req.Limit))}, {Key: "offset", Value: attribute.IntValue(int(req.Offset))}, {Key: "with_total", Value: attribute.BoolValue(req.WithTotal)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Source != nil { attributes = append(attributes, attribute.KeyValue{Key: "source", Value: attribute.StringValue(*req.Source)}) @@ -37,7 +39,7 @@ func (a *API) GetTopGroups(ctx context.Context, req *errorgroups.GetTopGroupsReq span.SetAttributes(attributes...) request := types.GetTopErrorGroupsRequest{ - Env: req.Env, + Env: reqEnv, Source: req.Source, TimeRange: parseTimeRange(req), Limit: req.Limit, @@ -45,7 +47,7 @@ func (a *API) GetTopGroups(ctx context.Context, req *errorgroups.GetTopGroupsReq WithTotal: req.WithTotal, } - groups, total, err := a.service.GetTopErrorGroups(ctx, request) + groups, total, err := svc.GetTopErrorGroups(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_top_groups_test.go b/internal/api/errorgroups/v1/grpc/get_top_groups_test.go index c276f1e..4297cdb 100644 --- a/internal/api/errorgroups/v1/grpc/get_top_groups_test.go +++ b/internal/api/errorgroups/v1/grpc/get_top_groups_test.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -239,7 +240,7 @@ func TestGetTopGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/api.go b/internal/api/errorgroups/v1/http/api.go index 71711d1..3ca9378 100644 --- a/internal/api/errorgroups/v1/http/api.go +++ b/internal/api/errorgroups/v1/http/api.go @@ -1,6 +1,7 @@ package http import ( + "context" "strconv" "time" @@ -11,15 +12,23 @@ import ( ) type API struct { - service errorgroups.Service + services map[string]errorgroups.Service } -func New(svc errorgroups.Service) *API { +func New(services map[string]errorgroups.Service) *API { return &API{ - service: svc, + services: services, } } +func (a *API) serviceByEnv(ctx context.Context, reqEnv *string) (errorgroups.Service, *string) { + env := types.GetEnvKey(ctx) + if (reqEnv == nil || *reqEnv == "") && env != "" { + reqEnv = &env + } + return a.services[env], reqEnv +} + func (a *API) Router() chi.Router { mux := chi.NewMux() diff --git a/internal/api/errorgroups/v1/http/diff_by_releases.go b/internal/api/errorgroups/v1/http/diff_by_releases.go index c6e2299..d7f7e0b 100644 --- a/internal/api/errorgroups/v1/http/diff_by_releases.go +++ b/internal/api/errorgroups/v1/http/diff_by_releases.go @@ -19,6 +19,7 @@ import ( // @Router /errorgroups/v1/diff_by_releases [post] // @ID errorgroups_v1_diff_by_releases // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body diffByReleasesRequest true "Request body" // @Success 200 {object} diffByReleasesResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -35,6 +36,8 @@ func (a *API) serveDiffByReleases(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, {Key: "releases", Value: attribute.StringSliceValue(httpReq.Releases)}, @@ -43,18 +46,18 @@ func (a *API) serveDiffByReleases(w http.ResponseWriter, r *http.Request) { {Key: "order", Value: attribute.StringValue(string(httpReq.Order))}, {Key: "with_total", Value: attribute.BoolValue(httpReq.WithTotal)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Source != nil { attributes = append(attributes, attribute.KeyValue{Key: "source", Value: attribute.StringValue(*httpReq.Source)}) } span.SetAttributes(attributes...) - groups, total, err := a.service.DiffByReleases(ctx, types.DiffByReleasesRequest{ + groups, total, err := svc.DiffByReleases(ctx, types.DiffByReleasesRequest{ Service: httpReq.Service, Releases: httpReq.Releases, - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, Limit: httpReq.Limit, Offset: httpReq.Offset, diff --git a/internal/api/errorgroups/v1/http/diff_by_releases_test.go b/internal/api/errorgroups/v1/http/diff_by_releases_test.go index 8e27968..1e09d1e 100644 --- a/internal/api/errorgroups/v1/http/diff_by_releases_test.go +++ b/internal/api/errorgroups/v1/http/diff_by_releases_test.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -144,7 +145,7 @@ func TestServeDiffByReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_details.go b/internal/api/errorgroups/v1/http/get_details.go index 39d0156..3b8759a 100644 --- a/internal/api/errorgroups/v1/http/get_details.go +++ b/internal/api/errorgroups/v1/http/get_details.go @@ -20,6 +20,7 @@ import ( // @Router /errorgroups/v1/details [post] // @ID errorgroups_v1_get_details // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getDetailsRequest true "Request body" // @Success 200 {object} getDetailsResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -36,6 +37,8 @@ func (a *API) serveGetDetails(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + parsedGroupHash, err := parseGroupHash(&httpReq.GroupHash) if err != nil { wr.Error(fmt.Errorf("failed to parse group_hash: %w", err), http.StatusBadRequest) @@ -45,8 +48,8 @@ func (a *API) serveGetDetails(w http.ResponseWriter, r *http.Request) { attributes := []attribute.KeyValue{ {Key: "group_hash", Value: attribute.StringValue(httpReq.GroupHash)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*httpReq.Service)}) @@ -65,7 +68,7 @@ func (a *API) serveGetDetails(w http.ResponseWriter, r *http.Request) { req := types.GetErrorGroupDetailsRequest{ GroupHash: *parsedGroupHash, - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, Service: httpReq.Service, Release: httpReq.Release, @@ -77,7 +80,7 @@ func (a *API) serveGetDetails(w http.ResponseWriter, r *http.Request) { } } - details, err := a.service.GetDetails(ctx, req) + details, err := svc.GetDetails(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_details_test.go b/internal/api/errorgroups/v1/http/get_details_test.go index 88fb479..a954774 100644 --- a/internal/api/errorgroups/v1/http/get_details_test.go +++ b/internal/api/errorgroups/v1/http/get_details_test.go @@ -11,6 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -260,7 +261,7 @@ func TestServeGetDetails(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_groups.go b/internal/api/errorgroups/v1/http/get_groups.go index 41c2ddb..15580d6 100644 --- a/internal/api/errorgroups/v1/http/get_groups.go +++ b/internal/api/errorgroups/v1/http/get_groups.go @@ -19,6 +19,7 @@ import ( // @Router /errorgroups/v1/groups [post] // @ID errorgroups_v1_get_groups // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getGroupsRequest true "Request body" // @Success 200 {object} getGroupsResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -35,6 +36,8 @@ func (a *API) serveGetGroups(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, {Key: "limit", Value: attribute.IntValue(int(httpReq.Limit))}, @@ -42,8 +45,8 @@ func (a *API) serveGetGroups(w http.ResponseWriter, r *http.Request) { {Key: "order", Value: attribute.StringValue(string(httpReq.Order))}, {Key: "with_total", Value: attribute.BoolValue(httpReq.WithTotal)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*httpReq.Release)}) @@ -72,7 +75,7 @@ func (a *API) serveGetGroups(w http.ResponseWriter, r *http.Request) { req := types.GetErrorGroupsRequest{ Service: httpReq.Service, - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, Release: httpReq.Release, TimeRange: tr, @@ -93,9 +96,9 @@ func (a *API) serveGetGroups(w http.ResponseWriter, r *http.Request) { total uint64 ) if httpReq.Filter != nil && httpReq.Filter.IsNew { - groups, total, err = a.service.GetNewErrorGroups(ctx, req) + groups, total, err = svc.GetNewErrorGroups(ctx, req) } else { - groups, total, err = a.service.GetErrorGroups(ctx, req) + groups, total, err = svc.GetErrorGroups(ctx, req) } if err != nil { diff --git a/internal/api/errorgroups/v1/http/get_groups_test.go b/internal/api/errorgroups/v1/http/get_groups_test.go index 2c532b1..56ef5a5 100644 --- a/internal/api/errorgroups/v1/http/get_groups_test.go +++ b/internal/api/errorgroups/v1/http/get_groups_test.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -439,7 +440,7 @@ func TestServeGetGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { if tt.req.Filter != nil && tt.req.Filter.IsNew { diff --git a/internal/api/errorgroups/v1/http/get_hist.go b/internal/api/errorgroups/v1/http/get_hist.go index 47919b9..b3752ca 100644 --- a/internal/api/errorgroups/v1/http/get_hist.go +++ b/internal/api/errorgroups/v1/http/get_hist.go @@ -18,6 +18,7 @@ import ( // @Router /errorgroups/v1/hist [post] // @ID errorgroups_v1_get_hist // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getHistRequest true "Request body" // @Success 200 {object} getHistResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -34,6 +35,8 @@ func (a *API) serveGetHist(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{} if httpReq.GroupHash != nil { attributes = append(attributes, attribute.KeyValue{Key: "group_hash", Value: attribute.StringValue(*httpReq.GroupHash)}) @@ -41,8 +44,8 @@ func (a *API) serveGetHist(w http.ResponseWriter, r *http.Request) { if httpReq.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*httpReq.Service)}) } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*httpReq.Release)}) @@ -78,7 +81,7 @@ func (a *API) serveGetHist(w http.ResponseWriter, r *http.Request) { req := types.GetErrorHistRequest{ Service: httpReq.Service, GroupHash: parsedGroupHash, - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, Release: httpReq.Release, TimeRange: tr, @@ -90,7 +93,7 @@ func (a *API) serveGetHist(w http.ResponseWriter, r *http.Request) { } } - hist, err := a.service.GetHist(ctx, req) + hist, err := svc.GetHist(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_hist_test.go b/internal/api/errorgroups/v1/http/get_hist_test.go index 249fa40..7692410 100644 --- a/internal/api/errorgroups/v1/http/get_hist_test.go +++ b/internal/api/errorgroups/v1/http/get_hist_test.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -253,7 +254,7 @@ func TestServeGetHist(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_releases.go b/internal/api/errorgroups/v1/http/get_releases.go index 2a49974..2a79f5a 100644 --- a/internal/api/errorgroups/v1/http/get_releases.go +++ b/internal/api/errorgroups/v1/http/get_releases.go @@ -17,6 +17,7 @@ import ( // @Router /errorgroups/v1/releases [post] // @ID errorgroups_v1_get_releases // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getReleasesRequest true "Request body" // @Success 200 {object} getReleasesResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -33,19 +34,21 @@ func (a *API) serveGetReleases(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } span.SetAttributes(attributes...) req := types.GetReleasesRequest{ Service: httpReq.Service, - Env: httpReq.Env, + Env: httpReqEnv, } - releases, err := a.service.GetReleases(ctx, req) + releases, err := svc.GetReleases(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_releases_test.go b/internal/api/errorgroups/v1/http/get_releases_test.go index a603b92..4401946 100644 --- a/internal/api/errorgroups/v1/http/get_releases_test.go +++ b/internal/api/errorgroups/v1/http/get_releases_test.go @@ -9,6 +9,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -76,7 +77,7 @@ func TestServeGetReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_service_filters.go b/internal/api/errorgroups/v1/http/get_service_filters.go index 233d367..3b27e52 100644 --- a/internal/api/errorgroups/v1/http/get_service_filters.go +++ b/internal/api/errorgroups/v1/http/get_service_filters.go @@ -17,6 +17,7 @@ import ( // @Router /errorgroups/v1/service_filters [post] // @ID errorgroups_v1_get_service_filters // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getServiceFiltersRequest true "Request body" // @Success 200 {object} getServiceFiltersResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -33,19 +34,21 @@ func (a *API) serveGetServiceFilters(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } span.SetAttributes(attributes...) req := types.GetServiceFiltersRequest{ Service: httpReq.Service, - Env: httpReq.Env, + Env: httpReqEnv, } - filters, err := a.service.GetServiceFilters(ctx, req) + filters, err := svc.GetServiceFilters(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_service_filters_test.go b/internal/api/errorgroups/v1/http/get_service_filters_test.go index ecbfdec..80909cf 100644 --- a/internal/api/errorgroups/v1/http/get_service_filters_test.go +++ b/internal/api/errorgroups/v1/http/get_service_filters_test.go @@ -9,6 +9,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -82,7 +83,7 @@ func TestServeGetServiceFilters(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_services.go b/internal/api/errorgroups/v1/http/get_services.go index e02eea8..f630b1f 100644 --- a/internal/api/errorgroups/v1/http/get_services.go +++ b/internal/api/errorgroups/v1/http/get_services.go @@ -17,6 +17,7 @@ import ( // @Router /errorgroups/v1/services [post] // @ID errorgroups_v1_get_services // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getServicesRequest true "Request body" // @Success 200 {object} getServicesResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -33,21 +34,23 @@ func (a *API) serveGetServices(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "query", Value: attribute.StringValue(httpReq.Query)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } span.SetAttributes(attributes...) req := types.GetServicesRequest{ Query: httpReq.Query, - Env: httpReq.Env, + Env: httpReqEnv, Limit: httpReq.Limit, Offset: httpReq.Offset, } - services, err := a.service.GetServices(ctx, req) + services, err := svc.GetServices(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_services_test.go b/internal/api/errorgroups/v1/http/get_services_test.go index bd63165..f68abc0 100644 --- a/internal/api/errorgroups/v1/http/get_services_test.go +++ b/internal/api/errorgroups/v1/http/get_services_test.go @@ -9,6 +9,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -79,7 +80,7 @@ func TestServeGetServices(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_top_groups.go b/internal/api/errorgroups/v1/http/get_top_groups.go index c04d65d..ea009e6 100644 --- a/internal/api/errorgroups/v1/http/get_top_groups.go +++ b/internal/api/errorgroups/v1/http/get_top_groups.go @@ -18,6 +18,7 @@ import ( // @Router /errorgroups/v1/top_groups [post] // @ID errorgroups_v1_get_top_groups // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getTopGroupsRequest true "Request body" // @Success 200 {object} getTopGroupsResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -34,13 +35,15 @@ func (a *API) serveGetTopGroups(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "limit", Value: attribute.IntValue(int(httpReq.Limit))}, {Key: "offset", Value: attribute.IntValue(int(httpReq.Offset))}, {Key: "with_total", Value: attribute.BoolValue(httpReq.WithTotal)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Source != nil { attributes = append(attributes, attribute.KeyValue{Key: "source", Value: attribute.StringValue(*httpReq.Source)}) @@ -61,7 +64,7 @@ func (a *API) serveGetTopGroups(w http.ResponseWriter, r *http.Request) { } req := types.GetTopErrorGroupsRequest{ - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, TimeRange: tr, Limit: httpReq.Limit, @@ -69,7 +72,7 @@ func (a *API) serveGetTopGroups(w http.ResponseWriter, r *http.Request) { WithTotal: httpReq.WithTotal, } - groups, total, err := a.service.GetTopErrorGroups(ctx, req) + groups, total, err := svc.GetTopErrorGroups(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_top_groups_test.go b/internal/api/errorgroups/v1/http/get_top_groups_test.go index 7737259..546b6cb 100644 --- a/internal/api/errorgroups/v1/http/get_top_groups_test.go +++ b/internal/api/errorgroups/v1/http/get_top_groups_test.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -236,7 +237,7 @@ func TestServeGetTopGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index 30bf509..080dc7d 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -6,6 +6,8 @@ import ( "strconv" "time" + "go.uber.org/zap" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" @@ -14,7 +16,6 @@ import ( asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" - "go.uber.org/zap" ) type apiParams struct { diff --git a/internal/api/seqapi/v1/http/cancel_async_search.go b/internal/api/seqapi/v1/http/cancel_async_search.go index f2c2a91..f037a85 100644 --- a/internal/api/seqapi/v1/http/cancel_async_search.go +++ b/internal/api/seqapi/v1/http/cancel_async_search.go @@ -19,6 +19,7 @@ import ( // @ID seqapi_v1_cancel_async_search // @Tags seqapi_v1 // @Param id path string true "search id" +// @Param env query string false "Environment" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer diff --git a/internal/api/seqapi/v1/http/delete_async_search.go b/internal/api/seqapi/v1/http/delete_async_search.go index cd9b6dc..406d78a 100644 --- a/internal/api/seqapi/v1/http/delete_async_search.go +++ b/internal/api/seqapi/v1/http/delete_async_search.go @@ -19,6 +19,7 @@ import ( // @ID seqapi_v1_delete_async_search // @Tags seqapi_v1 // @Param id path string true "search id" +// @Param env query string false "Environment" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index 4878b8b..b1a3d3b 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -113,7 +113,7 @@ func TestServeGetEvent(t *testing.T) { }, }, } - + // убрать фатал при пинге(пинговать, но не фаталиться) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() diff --git a/internal/api/seqapi/v1/http/fetch_async_search_result.go b/internal/api/seqapi/v1/http/fetch_async_search_result.go index 24b0a13..14fa485 100644 --- a/internal/api/seqapi/v1/http/fetch_async_search_result.go +++ b/internal/api/seqapi/v1/http/fetch_async_search_result.go @@ -20,6 +20,7 @@ import ( // @Router /seqapi/v1/async_search/fetch [post] // @ID seqapi_v1_fetch_async_search_result // @Tags seqapi_v1 +// @Param env query string false "Environment" // @Param body body fetchAsyncSearchResultRequest true "Request body" // @Success 200 {object} fetchAsyncSearchResultResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index 5d86035..a8c5de4 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "google.golang.org/protobuf/proto" @@ -14,7 +15,6 @@ import ( mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" - "github.com/stretchr/testify/require" ) var fieldsCacheKey = "seqapi_fields" diff --git a/internal/api/seqapi/v1/http/get_async_searches_list.go b/internal/api/seqapi/v1/http/get_async_searches_list.go index a0b0c2c..efdc025 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list.go @@ -20,6 +20,7 @@ import ( // @Router /seqapi/v1/async_search/list [post] // @ID seqapi_v1_get_async_searches_list // @Tags seqapi_v1 +// @Param env query string false "Environment" // @Param body body getAsyncSearchesListRequest true "Request body" // @Success 200 {object} getAsyncSearchesListResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index 3602923..31543e0 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -22,6 +22,7 @@ import ( // @Router /seqapi/v1/async_search/start [post] // @ID seqapi_v1_start_async_search // @Tags seqapi_v1 +// @Param env query string false "Environment" // @Param body body startAsyncSearchRequest true "Request body" // @Success 200 {object} startAsyncSearchResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index d565be9..82d337d 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -1,15 +1,12 @@ package seqapi_v1 import ( - "context" - "github.com/go-chi/chi/v5" grpc_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/grpc" http_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/http" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" - "github.com/ozontech/seq-ui/internal/pkg/repository" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) @@ -18,15 +15,7 @@ type SeqAPI struct { httpAPI *http_api.API } -func New( - ctx context.Context, - cfg *config.SeqAPI, - seqDBClients map[string]seqdb.Client, - asyncSearhCfg *config.AsyncSearch, - asyncSearchesRepo repository.AsyncSearches, -) *SeqAPI { - asyncSearches := newAsyncSearches(ctx, asyncSearhCfg, asyncSearchesRepo, seqDBClients) - +func New(cfg *config.SeqAPI, seqDBClients map[string]seqdb.Client, asyncSearches map[string]asyncsearches.Service) *SeqAPI { return &SeqAPI{ grpcAPI: grpc_api.New(cfg, seqDBClients, asyncSearches), httpAPI: http_api.New(cfg, seqDBClients, asyncSearches), @@ -40,25 +29,3 @@ func (s *SeqAPI) GRPCServer() *grpc_api.API { func (s *SeqAPI) HTTPRouter() chi.Router { return s.httpAPI.Router() } - -func newAsyncSearches( - ctx context.Context, - cfg *config.AsyncSearch, - repo repository.AsyncSearches, - seqDBClients map[string]seqdb.Client, -) map[string]asyncsearches.Service { - services := make(map[string]asyncsearches.Service, max(1, len(cfg.Envs))) - if len(cfg.Envs) > 0 { - for envName, envCfg := range cfg.Envs { - services[envName] = asyncsearches.New(ctx, repo, seqDBClients[envCfg.SeqDBID], envCfg.Options, envName) - } - } else { - var client seqdb.Client - for _, c := range seqDBClients { - client = c - } - services[""] = asyncsearches.New(ctx, repo, client, &cfg.Options, "") - } - - return services -} diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index df613a7..dceccc1 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -28,7 +28,7 @@ func (m *v1tov2) migrate() v2.Config { dst.Cache = m.migrateCache(v1Cache) dst.Clients = m.migrateClients(m.src) dst.DB = m.migrateDB(m.src.Server) - // dst.Handlers = m.migrateHandlers(m.src.Handlers, &dst, v1Cache) + dst.Handlers = m.migrateHandlers(m.src.Handlers, &dst, v1Cache) return dst } @@ -285,184 +285,160 @@ func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { } } -// func (m *v1tov2) migrateHandlers(src *v1.Handlers, cfg *v2.Config, v1Cache v1.Cache) v2.Handlers { -// if src == nil { -// return v2.Handlers{} -// } -// -// dst := v2.Handlers{ -// SeqAPI: m.migrateSeqAPI(src.SeqAPI, v1Cache), -// ErrorGroups: m.migrateErrorGroups(src.ErrorGroups), -// AsyncSearch: m.migrateAsyncSearch(src.AsyncSearch), -// Admin: m.migrateAdmin(src.Admin, v1Cache), -// } -// -// if src.MassExport != nil { -// dst.MassExport = m.migrateMassExport(src.MassExport, cfg) -// } -// -// return dst -// } -// -// func (m *v1tov2) migrateSeqAPI(src v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { -// cacheID, redisID := m.defaultCacheIDs(v1Cache) -// dst := v2.SeqAPI{ -// Envs: m.migrateSeqAPIEnvs(src.Envs, cacheID, redisID), -// DefaultEnv: src.DefaultEnv, -// Options: v2.SeqAPIOptions{ -// Caches: v2.SeqAPICaches{ -// Events: v2.HandlerCache{ -// ID: cacheID, -// TTL: src.EventsCacheTTL, -// }, -// LogsLifespan: v2.HandlerCache{ -// ID: redisID, -// TTL: src.LogsLifespanCacheTTL, -// }, -// Fields: v2.HandlerCache{ -// ID: cacheID, -// TTL: src.FieldsCacheTTL, -// }, -// }, -// }, -// } -// -// if len(src.Envs) == 0 { -// dst.SeqDBID = v2.DefaultSeqDBClientID -// } -// -// if src.SeqAPIOptions != nil { -// dst.Options = m.migrateSeqAPIOptions(src.SeqAPIOptions, cacheID, redisID) -// } -// -// return dst -// } -// -// func (m *v1tov2) migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv, cacheID, redisID string) map[string]v2.SeqAPIEnv { -// if len(envs) == 0 { -// return nil -// } -// -// dst := make(map[string]v2.SeqAPIEnv, len(envs)) -// for name, cfg := range envs { -// env := v2.SeqAPIEnv{ -// SeqDBID: cfg.SeqDB, -// } -// if cfg.Options != nil { -// envOptions := m.migrateSeqAPIOptions(cfg.Options, cacheID, redisID) -// env.Options = &envOptions -// } -// -// dst[name] = env -// } -// -// return dst -// } -// -// func (m *v1tov2) migrateSeqAPIOptions(options *v1.SeqAPIOptions, cacheID, redisID string) v2.SeqAPIOptions { -// return v2.SeqAPIOptions{ -// Limits: v2.SeqAPILimits{ -// MaxSearchLimit: options.MaxSearchLimit, -// MaxSearchTotal: options.MaxSearchTotalLimit, -// MaxSearchOffset: options.MaxSearchOffsetLimit, -// MaxExportLimit: options.MaxExportLimit, -// SeqCLIMaxSearchLimit: options.SeqCLIMaxSearchLimit, -// MaxParallelExportRequests: options.MaxParallelExportRequests, -// MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, -// MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, -// }, -// Masking: m.migrateMasking(options.Masking), -// PinnedFields: m.migrateFields(options.PinnedFields), -// SystemFields: m.migrateFields(options.SystemFields), -// Caches: v2.SeqAPICaches{ -// Events: v2.HandlerCache{ -// ID: cacheID, -// TTL: options.EventsCacheTTL, -// }, -// LogsLifespan: v2.HandlerCache{ -// ID: redisID, -// TTL: options.LogsLifespanCacheTTL, -// }, -// Fields: v2.HandlerCache{ -// ID: cacheID, -// TTL: options.FieldsCacheTTL, -// }, -// }, -// } -// } -// -// func (m *v1tov2) migrateFields(fs []v1.Field) []v2.Field { -// if fs == nil { -// return nil -// } -// -// dst := make([]v2.Field, len(fs)) -// for i, f := range fs { -// dst[i] = v2.Field{ -// Name: f.Name, -// Type: f.Type, -// } -// } -// -// return dst -// } -// -// func (m *v1tov2) migrateMasking(src *v1.Masking) *v2.Masking { -// if src == nil { -// return nil -// } -// -// return &v2.Masking{ -// Masks: m.migrateMasks(src.Masks), -// ProcessFields: src.ProcessFields, -// IgnoreFields: src.IgnoreFields, -// } -// } -// -// func (m *v1tov2) migrateMasks(ms []v1.Mask) []v2.Mask { -// if ms == nil { -// return nil -// } -// -// dst := make([]v2.Mask, len(ms)) -// for i := range ms { -// mask := &ms[i] -// dst[i] = v2.Mask{ -// Re: mask.Re, -// Groups: mask.Groups, -// Mode: mask.Mode, -// ReplaceWord: mask.ReplaceWord, -// ProcessFields: mask.ProcessFields, -// IgnoreFields: mask.IgnoreFields, -// FieldFilters: m.migrateFieldFilters(mask.FieldFilters), -// } -// } -// -// return dst -// } -// -// func (m *v1tov2) migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { -// if src == nil { -// return nil -// } -// -// dst := &v2.FieldFilterSet{ -// Condition: src.Condition, -// } -// if src.Filters != nil { -// dst.Filters = make([]v2.FieldFilter, len(src.Filters)) -// for i, f := range src.Filters { -// dst.Filters[i] = v2.FieldFilter{ -// Field: f.Field, -// Mode: f.Mode, -// Values: f.Values, -// } -// } -// } -// -// return dst -// } -// +func (m *v1tov2) migrateHandlers(src *v1.Handlers, v1Cache v1.Cache) v2.Handlers { + if src == nil { + src = &v1.Handlers{} + } + + dst := v2.Handlers{ + SeqAPI: m.migrateSeqAPI(&src.SeqAPI, v1Cache), + AsyncSearch: m.migrateAsyncSearch(&src.AsyncSearch, &src.SeqAPI), + // ErrorGroups: m.migrateErrorGroups(src.ErrorGroups), + // Admin: m.migrateAdmin(src.Admin, v1Cache), + } + + // if src.MassExport != nil { + // dst.MassExport = m.migrateMassExport(src.MassExport, cfg) + // } + + return dst +} + +func (m *v1tov2) migrateSeqAPI(src *v1.SeqAPI, v1Cache v1.Cache) *v2.SeqAPI { + dst := &v2.SeqAPI{ + Options: m.migrateSeqAPIOptions(src.SeqAPIOptions, v1Cache), + DefaultEnv: src.DefaultEnv, + } + + if len(src.Envs) > 0 { + dst.Envs = make(map[string]v2.SeqAPIEnv, len(src.Envs)) + for envName, envCfg := range src.Envs { + dstEnv := v2.SeqAPIEnv{SeqDBID: envCfg.SeqDB} + if envCfg.Options != nil { + options := m.migrateSeqAPIOptions(envCfg.Options, v1Cache) + dstEnv.Options = &options + } + dst.Envs[envName] = dstEnv + } + } + + return dst +} + +func (m *v1tov2) migrateSeqAPIOptions(src *v1.SeqAPIOptions, v1Cache v1.Cache) v2.SeqAPIOptions { + if src == nil { + src = &v1.SeqAPIOptions{} + } + + redisWithInmemID, redisOrInmemID := m.defaultCacheIDs(v1Cache) + dst := v2.SeqAPIOptions{ + Limits: v2.SeqAPILimits{ + MaxSearchLimit: src.MaxSearchLimit, + MaxSearchTotal: src.MaxSearchTotalLimit, + MaxSearchOffset: src.MaxSearchOffsetLimit, + MaxExportLimit: src.MaxExportLimit, + SeqCLIMaxSearchLimit: src.SeqCLIMaxSearchLimit, + MaxParallelExportRequests: src.MaxParallelExportRequests, + MaxAggregationsPerRequest: src.MaxAggregationsPerRequest, + MaxBucketsPerAggregationTs: src.MaxBucketsPerAggregationTs, + }, + Masking: m.migrateMasking(src.Masking), + PinnedFields: m.migrateFields(src.PinnedFields), + SystemFields: m.migrateFields(src.SystemFields), + Caches: v2.SeqAPICaches{ + Events: &v2.HandlerCache{ + ID: redisWithInmemID, + TTL: src.EventsCacheTTL, + }, + LogsLifespan: &v2.HandlerCache{ + ID: redisOrInmemID, + TTL: src.LogsLifespanCacheTTL, + }, + }, + } + + if src.FieldsCacheTTL > 0 { + dst.Caches.Fields = &v2.HandlerCache{ + ID: v2.DefaultInmemID, + TTL: src.FieldsCacheTTL, + } + } + + return dst +} + +func (m *v1tov2) migrateFields(fs []v1.Field) []v2.Field { + if fs == nil { + return nil + } + + dst := make([]v2.Field, len(fs)) + for i, f := range fs { + dst[i] = v2.Field{ + Name: f.Name, + Type: f.Type, + } + } + + return dst +} + +func (m *v1tov2) migrateMasking(src *v1.Masking) *v2.Masking { + if src == nil { + return nil + } + + return &v2.Masking{ + Masks: m.migrateMasks(src.Masks), + ProcessFields: src.ProcessFields, + IgnoreFields: src.IgnoreFields, + } +} + +func (m *v1tov2) migrateMasks(ms []v1.Mask) []v2.Mask { + if ms == nil { + return nil + } + + dst := make([]v2.Mask, len(ms)) + for i := range ms { + mask := &ms[i] + dst[i] = v2.Mask{ + Re: mask.Re, + Groups: mask.Groups, + Mode: mask.Mode, + ReplaceWord: mask.ReplaceWord, + ProcessFields: mask.ProcessFields, + IgnoreFields: mask.IgnoreFields, + FieldFilters: m.migrateFieldFilters(mask.FieldFilters), + } + } + + return dst +} + +func (m *v1tov2) migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { + if src == nil { + return nil + } + + dst := &v2.FieldFilterSet{ + Condition: src.Condition, + } + if src.Filters != nil { + dst.Filters = make([]v2.FieldFilter, len(src.Filters)) + for i, f := range src.Filters { + dst.Filters[i] = v2.FieldFilter{ + Field: f.Field, + Mode: f.Mode, + Values: f.Values, + } + } + } + + return dst +} + // func (m *v1tov2) migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { // if m.isEgEmpty(eg) { // return nil @@ -480,21 +456,25 @@ func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { // }, // } // } -// -// func (m *v1tov2) migrateAsyncSearch(a v1.AsyncSearch) *v2.AsyncSearch { -// if len(a.AdminUsers) == 0 && a.ListQueryLengthLimit == 0 { -// return nil -// } -// -// return &v2.AsyncSearch{ -// SeqDBID: v2.DefaultSeqDBClientID, -// Options: v2.AsyncSearchOptions{ -// AdminUsers: a.AdminUsers, -// ListQueryLengthLimit: a.ListQueryLengthLimit, -// }, -// } -// } -// +func (m *v1tov2) migrateAsyncSearch(src *v1.AsyncSearch, seqAPI *v1.SeqAPI) *v2.AsyncSearch { + dst := &v2.AsyncSearch{ + Options: v2.AsyncSearchOptions{ + AdminUsers: src.AdminUsers, + ListQueryLengthLimit: src.ListQueryLengthLimit, + }, + DefaultEnv: seqAPI.DefaultEnv, + } + + if len(seqAPI.Envs) > 0 { + dst.Envs = make(map[string]v2.AsyncSearchEnv, len(seqAPI.Envs)) + for envName, envCfg := range seqAPI.Envs { + dst.Envs[envName] = v2.AsyncSearchEnv{SeqDBID: envCfg.SeqDB} + } + } + + return dst +} + // func (m *v1tov2) migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { // dst := &v2.MassExport{ // SeqDBID: v2.DefaultSeqDBClientID, @@ -573,7 +553,7 @@ func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { // } // // defaultCacheIDs returns cache ids same as in v1. -// inmemWithRedisID - inmemory+redis or inmemory (v1 events, oidc, fields) +// inmemWithRedisID - inmemory+redis or inmemory (v1 events, oidc) // redisOrInmemID - redis or inmemory (v1 logslifespan) func (m *v1tov2) defaultCacheIDs(src v1.Cache) (inmemWithRedisID, redisOrInmemID string) { if src.Redis != nil { diff --git a/internal/app/types/admin.go b/internal/app/types/admin.go index bf4b9b5..0cd8b32 100644 --- a/internal/app/types/admin.go +++ b/internal/app/types/admin.go @@ -27,6 +27,7 @@ type CreateRoleResponse struct { type AddUsersToRoleRequest struct { RoleID int32 Usernames []string + Env string } type GetRolesResponse struct { @@ -35,6 +36,7 @@ type GetRolesResponse struct { type GetRoleRequest struct { RoleID int32 + Env string } type GetRoleResponse struct { @@ -50,10 +52,12 @@ type UpdateRoleRequest struct { type DeleteRoleRequest struct { RoleID int32 ReplacementRoleID *int32 + Env string } type GetUserPermissionsRequest struct { Username string + Env string } type GetUserPermissionsResponse struct { @@ -63,4 +67,5 @@ type GetUserPermissionsResponse struct { type DeleteUsersFromRoleRequest struct { RoleID int32 Usernames []string + Env string } diff --git a/internal/pkg/repository/mock/repository.go b/internal/pkg/repository/mock/repository.go index 8efcc30..5e3004d 100644 --- a/internal/pkg/repository/mock/repository.go +++ b/internal/pkg/repository/mock/repository.go @@ -290,7 +290,7 @@ func (m *MockAsyncSearches) EXPECT() *MockAsyncSearchesMockRecorder { } // DeleteAsyncSearch mocks base method. -func (m *MockAsyncSearches) DeleteAsyncSearch(arg0 context.Context, arg1 string) error { +func (m *MockAsyncSearches) DeleteAsyncSearch(arg0 context.Context, arg1 types.DeleteAsyncSearchRequest) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteAsyncSearch", arg0, arg1) ret0, _ := ret[0].(error) @@ -304,21 +304,21 @@ func (mr *MockAsyncSearchesMockRecorder) DeleteAsyncSearch(arg0, arg1 any) *gomo } // DeleteExpiredAsyncSearches mocks base method. -func (m *MockAsyncSearches) DeleteExpiredAsyncSearches(arg0 context.Context) error { +func (m *MockAsyncSearches) DeleteExpiredAsyncSearches(arg0 context.Context, arg1 types.DeleteExpiredAsyncSearchesRequest) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteExpiredAsyncSearches", arg0) + ret := m.ctrl.Call(m, "DeleteExpiredAsyncSearches", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // DeleteExpiredAsyncSearches indicates an expected call of DeleteExpiredAsyncSearches. -func (mr *MockAsyncSearchesMockRecorder) DeleteExpiredAsyncSearches(arg0 any) *gomock.Call { +func (mr *MockAsyncSearchesMockRecorder) DeleteExpiredAsyncSearches(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteExpiredAsyncSearches", reflect.TypeOf((*MockAsyncSearches)(nil).DeleteExpiredAsyncSearches), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteExpiredAsyncSearches", reflect.TypeOf((*MockAsyncSearches)(nil).DeleteExpiredAsyncSearches), arg0, arg1) } // GetAsyncSearchById mocks base method. -func (m *MockAsyncSearches) GetAsyncSearchById(arg0 context.Context, arg1 string) (types.AsyncSearchInfo, error) { +func (m *MockAsyncSearches) GetAsyncSearchById(arg0 context.Context, arg1 types.GetAsyncSearchByIdRequest) (types.AsyncSearchInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAsyncSearchById", arg0, arg1) ret0, _ := ret[0].(types.AsyncSearchInfo) diff --git a/internal/pkg/service/admin/cache.go b/internal/pkg/service/admin/cache.go index cb9ebba..5fb9035 100644 --- a/internal/pkg/service/admin/cache.go +++ b/internal/pkg/service/admin/cache.go @@ -22,12 +22,18 @@ const ( var errCacheDisabled = errors.New("admin cache disabled") type adminCache struct { - cache cache.Cache - ttl time.Duration + cache cache.Cache + userPermsKeyPrefix string + ttl time.Duration } -func newAdminCache(c cache.Cache, ttl time.Duration) adminCache { - return adminCache{cache: c, ttl: ttl} +func newAdminCache(c cache.Cache, ttl time.Duration, env string) adminCache { + userPermsKeyPrefix := cacheKeyUserPerms + if env != "" { + userPermsKeyPrefix += env + "_" + } + + return adminCache{cache: c, ttl: ttl, userPermsKeyPrefix: userPermsKeyPrefix} } func (c *adminCache) getRoles(ctx context.Context) ([]types.Role, error) { @@ -77,7 +83,7 @@ func (c *adminCache) getUserPermissions(ctx context.Context, username string) ([ return nil, errCacheDisabled } - data, err := c.cache.Get(ctx, cacheKeyUserPerms+username) + data, err := c.cache.Get(ctx, c.userPermsKeyPrefix+username) if err != nil { return nil, fmt.Errorf("failed to get user permissions from cache: %w", err) } @@ -101,7 +107,7 @@ func (c *adminCache) setUserPermissions(ctx context.Context, username string, pe return } - if err := c.cache.SetWithTTL(ctx, cacheKeyUserPerms+username, string(data), c.ttl); err != nil { + if err := c.cache.SetWithTTL(ctx, c.userPermsKeyPrefix+username, string(data), c.ttl); err != nil { logger.Error("failed to set user permissions in cache", zap.Error(err)) } } @@ -112,6 +118,6 @@ func (c *adminCache) resetUsersPermissions(ctx context.Context, usernames ...str } for _, username := range usernames { - c.cache.Del(ctx, cacheKeyUserPerms+username) + c.cache.Del(ctx, c.userPermsKeyPrefix+username) } } diff --git a/internal/pkg/service/admin/permissions.go b/internal/pkg/service/admin/permissions.go index 5d3eb15..3d27db2 100644 --- a/internal/pkg/service/admin/permissions.go +++ b/internal/pkg/service/admin/permissions.go @@ -34,7 +34,10 @@ func (s *service) checkAccess(ctx context.Context, requiredPermission string) er return nil } - permissions, err := s.getUserPermissions(ctx, types.GetUserPermissionsRequest{Username: username}) + permissions, err := s.getUserPermissions(ctx, types.GetUserPermissionsRequest{ + Username: username, + Env: s.env, + }) if err != nil { return fmt.Errorf("can't get user permissions: %w", err) } diff --git a/internal/pkg/service/admin/permissions_test.go b/internal/pkg/service/admin/permissions_test.go index a0922de..5294654 100644 --- a/internal/pkg/service/admin/permissions_test.go +++ b/internal/pkg/service/admin/permissions_test.go @@ -68,7 +68,7 @@ func TestCheckAccess(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg).(*service) + svc := New(adminRepo, upRepo, cache, adminCfg, "").(*service) if tt.mockArgs != nil { cache.EXPECT(). @@ -80,7 +80,7 @@ func TestCheckAccess(t *testing.T) { Return(tt.mockArgs.perms, tt.mockArgs.err). Times(1) cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.CacheTTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -152,7 +152,7 @@ func TestGetUserPermissions(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg).(*service) + svc := New(adminRepo, upRepo, cache, adminCfg, "").(*service) cache.EXPECT(). Get(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username). @@ -167,7 +167,7 @@ func TestGetUserPermissions(t *testing.T) { if tt.mockArgs.err == nil { cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.CacheTTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -220,7 +220,7 @@ func TestValidatePermissions(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg).(*service) + svc := New(adminRepo, upRepo, cache, adminCfg, "").(*service) err := svc.validatePermissions(tt.perms) require.Equal(t, tt.wantErr, err != nil) diff --git a/internal/pkg/service/admin/roles.go b/internal/pkg/service/admin/roles.go index 2b5192c..83bea1b 100644 --- a/internal/pkg/service/admin/roles.go +++ b/internal/pkg/service/admin/roles.go @@ -60,6 +60,7 @@ func (s *service) AddUsersToRole(ctx context.Context, req types.AddUsersToRoleRe } } + req.Env = s.env if err := s.adminRepo.AddUsersToRole(ctx, req); err != nil { return err } @@ -86,6 +87,7 @@ func (s *service) DeleteUsersFromRole(ctx context.Context, req types.DeleteUsers return types.NewErrInvalidRequestField("empty username") } + req.Env = s.env if err := s.adminRepo.DeleteUsersFromRole(ctx, req); err != nil { return err } @@ -125,6 +127,7 @@ func (s *service) GetRole(ctx context.Context, req types.GetRoleRequest) (types. return types.RoleInfo{}, types.NewErrInvalidRequestField("value role_id must be greater than 0") } + req.Env = s.env return s.adminRepo.GetRole(ctx, req) } @@ -154,7 +157,7 @@ func (s *service) UpdateRole(ctx context.Context, req types.UpdateRoleRequest) e s.cache.resetRoles(ctx) if len(req.Permissions) > 0 { - if roleInfo, err := s.adminRepo.GetRole(ctx, types.GetRoleRequest{RoleID: req.RoleID}); err == nil { + if roleInfo, err := s.adminRepo.GetRole(ctx, types.GetRoleRequest{RoleID: req.RoleID, Env: s.env}); err == nil { s.cache.resetUsersPermissions(ctx, roleInfo.Usernames...) } } @@ -180,7 +183,7 @@ func (s *service) DeleteRole(ctx context.Context, req types.DeleteRoleRequest) e } } - if roleInfo, err := s.adminRepo.GetRole(ctx, types.GetRoleRequest{RoleID: req.RoleID}); err == nil { + if roleInfo, err := s.adminRepo.GetRole(ctx, types.GetRoleRequest{RoleID: req.RoleID, Env: s.env}); err == nil { s.cache.resetUsersPermissions(ctx, roleInfo.Usernames...) } diff --git a/internal/pkg/service/admin/roles_test.go b/internal/pkg/service/admin/roles_test.go index 5982b43..babfcf5 100644 --- a/internal/pkg/service/admin/roles_test.go +++ b/internal/pkg/service/admin/roles_test.go @@ -16,8 +16,13 @@ import ( ) var ( - defaultSuperUser = "superuser" - adminCfg = &config.Admin{SuperUsers: []string{defaultSuperUser}, CacheTTL: 5 * time.Minute} + defaultSuperUser = "superuser" + adminCfg = &config.AdminOptions{ + SuperUsers: []string{defaultSuperUser}, + Cache: &config.HandlerCache{ + TTL: 5 * time.Minute, + }, + } errSomethingWrong = errors.New("something happened wrong") ) @@ -46,7 +51,7 @@ func setupAccessMock(ctx context.Context, repo *mock.MockAdmin, cache *mock_cach Times(1) cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+access.username, gomock.Any(), adminCfg.CacheTTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+access.username, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -193,7 +198,7 @@ func TestCreateRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -340,7 +345,7 @@ func TestAddUsersToRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -458,7 +463,7 @@ func TestGetRoles(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -475,7 +480,7 @@ func TestGetRoles(t *testing.T) { if tt.mockArgs.err == nil { cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyRoles, gomock.Any(), adminCfg.CacheTTL). + SetWithTTL(gomock.Any(), cacheKeyRoles, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -570,7 +575,7 @@ func TestGetRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -781,7 +786,7 @@ func TestUpdateRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -939,7 +944,7 @@ func TestDeleteRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -1095,7 +1100,7 @@ func TestDeleteUsersFromRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) diff --git a/internal/pkg/service/admin/service.go b/internal/pkg/service/admin/service.go index c4e51d4..9d7d9e8 100644 --- a/internal/pkg/service/admin/service.go +++ b/internal/pkg/service/admin/service.go @@ -2,6 +2,7 @@ package admin import ( "context" + "time" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" @@ -25,18 +26,46 @@ type service struct { userProfilesRepo repository.UserProfiles cache adminCache superUsers map[string]struct{} + env string } -func New(adminRepo repository.Admin, userProfilesRepo repository.UserProfiles, c cache.Cache, cfg *config.Admin) Service { +func New(adminRepo repository.Admin, userProfilesRepo repository.UserProfiles, c cache.Cache, cfg *config.AdminOptions, env string) Service { su := make(map[string]struct{}, len(cfg.SuperUsers)) for _, u := range cfg.SuperUsers { su[u] = struct{}{} } + var ttl time.Duration + if cfg.Cache != nil { + ttl = cfg.Cache.TTL + } + return &service{ adminRepo: adminRepo, userProfilesRepo: userProfilesRepo, - cache: newAdminCache(c, cfg.CacheTTL), + cache: newAdminCache(c, ttl, env), superUsers: su, + env: env, + } +} + +func NewByEnv(cfg *config.Admin, adminRepo repository.Admin, userProfilesRepo repository.UserProfiles) map[string]Service { + services := make(map[string]Service, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envCfg := range cfg.Envs { + var c cache.Cache + if envCfg.Options.Cache != nil { + c, _ = cache.ByID(envCfg.Options.Cache.ID) + } + services[envName] = New(adminRepo, userProfilesRepo, c, envCfg.Options, envName) + } + } else { + var c cache.Cache + if cfg.Options.Cache != nil { + c, _ = cache.ByID(cfg.Options.Cache.ID) + } + services[""] = New(adminRepo, userProfilesRepo, c, &cfg.Options, "") } + + return services } diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index b185736..8af7bb0 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -58,6 +58,33 @@ func New(ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, return s } +// NewByEnv creates service for each env. +func NewByEnv( + ctx context.Context, + cfg *config.AsyncSearch, + repo repository.AsyncSearches, + seqDBClients map[string]seqdb.Client, +) map[string]Service { + if cfg == nil || repo == nil { + return nil + } + + services := make(map[string]Service, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envCfg := range cfg.Envs { + services[envName] = New(ctx, repo, seqDBClients[envCfg.SeqDBID], envCfg.Options, envName) + } + } else { + var client seqdb.Client + for _, c := range seqDBClients { + client = c + } + services[""] = New(ctx, repo, client, &cfg.Options, "") + } + + return services +} + func (s *service) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSearchRequest) (*seqapi.StartAsyncSearchResponse, error) { ownerID, err := profiles.GetIDFromContext(ctx) if err != nil { diff --git a/internal/pkg/service/errorgroups/service.go b/internal/pkg/service/errorgroups/service.go index f8d5047..d7fd96e 100644 --- a/internal/pkg/service/errorgroups/service.go +++ b/internal/pkg/service/errorgroups/service.go @@ -7,6 +7,7 @@ import ( "math" "slices" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "golang.org/x/sync/errgroup" "github.com/ozontech/seq-ui/internal/app/config/v2" @@ -46,6 +47,25 @@ func New(repo repositorych.Repository, logTagsMapping config.LogTagsMapping) Ser } } +func NewByEnv(cfg *config.ErrorGroups, clientsCfg *config.Clients, chClients map[string]driver.Conn) map[string]Service { + services := make(map[string]Service, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envCfg := range cfg.Envs { + repo := repositorych.New(chClients[envCfg.CHID], clientsCfg.ClickHouseByID(envCfg.CHID).Sharded, envCfg.Options.QueryFilter) + services[envName] = New(repo, envCfg.Options.LogTagsMapping) + } + } else { + var chID string + for id := range chClients { + chID = id + } + repo := repositorych.New(chClients[chID], clientsCfg.ClickHouseByID(chID).Sharded, cfg.Options.QueryFilter) + services[""] = New(repo, cfg.Options.LogTagsMapping) + } + + return services +} + func (s *service) GetErrorGroups( ctx context.Context, req types.GetErrorGroupsRequest, diff --git a/swagger/swagger.json b/swagger/swagger.json index 44b0a89..a812ae0 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -23,6 +23,14 @@ "admin_v1" ], "operationId": "admin_v1_get_roles", + "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + } + ], "responses": { "200": { "description": "A successful response", @@ -49,6 +57,12 @@ ], "operationId": "admin_v1_create_role", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -94,6 +108,12 @@ "name": "id", "in": "path", "required": true + }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" } ], "responses": { @@ -130,6 +150,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -170,6 +196,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -213,6 +245,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -254,6 +292,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -548,6 +592,12 @@ ], "operationId": "errorgroups_v1_get_details", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -586,6 +636,12 @@ ], "operationId": "errorgroups_v1_diff_by_releases", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -624,6 +680,12 @@ ], "operationId": "errorgroups_v1_get_groups", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -662,6 +724,12 @@ ], "operationId": "errorgroups_v1_get_hist", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -700,6 +768,12 @@ ], "operationId": "errorgroups_v1_get_releases", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -738,6 +812,12 @@ ], "operationId": "errorgroups_v1_get_service_filters", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -776,6 +856,12 @@ ], "operationId": "errorgroups_v1_get_services", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -814,6 +900,12 @@ ], "operationId": "errorgroups_v1_get_top_groups", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -1169,6 +1261,12 @@ ], "operationId": "seqapi_v1_fetch_async_search_result", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -1207,6 +1305,12 @@ ], "operationId": "seqapi_v1_get_async_searches_list", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -1245,6 +1349,12 @@ ], "operationId": "seqapi_v1_start_async_search", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -1289,6 +1399,12 @@ "name": "id", "in": "path", "required": true + }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" } ], "responses": { @@ -1322,6 +1438,12 @@ "name": "id", "in": "path", "required": true + }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" } ], "responses": { @@ -3500,24 +3622,19 @@ "type": "object", "properties": { "maxAggregationsPerRequest": { - "type": "integer", - "format": "int32" + "type": "integer" }, "maxExportLimit": { - "type": "integer", - "format": "int32" + "type": "integer" }, "maxParallelExportRequests": { - "type": "integer", - "format": "int32" + "type": "integer" }, "maxSearchLimit": { - "type": "integer", - "format": "int32" + "type": "integer" }, "seqCliMaxSearchLimit": { - "type": "integer", - "format": "int32" + "type": "integer" } } }, @@ -3589,12 +3706,10 @@ } }, "limit": { - "type": "integer", - "format": "int32" + "type": "integer" }, "offset": { - "type": "integer", - "format": "int32" + "type": "integer" }, "offset_id": { "type": "string" From 47368600d99891eba42b301a05e0358b2170c53e Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Mon, 19 Oct 2026 00:55:47 +0300 Subject: [PATCH 24/29] update repository/migration/main --- cmd/seq-ui/main.go | 84 ++++----- internal/app/config/migrate/v1_v2.go | 173 +++++++++--------- internal/app/config/migrate/v1_v2_test.go | 2 +- internal/pkg/client/seqdb/grpc_client.go | 4 +- internal/pkg/repository/admin_repo.go | 18 +- .../pkg/repository/async_searches_repo.go | 2 +- ...er_table_async_searches_add_column_env.sql | 9 - migration/13_add_column_env.sql | 27 +++ tracing/tracing.go | 19 +- 9 files changed, 173 insertions(+), 165 deletions(-) delete mode 100644 migration/12_alter_table_async_searches_add_column_env.sql create mode 100644 migration/13_add_column_env.sql diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 4cdf3b5..e4a4d9a 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -6,7 +6,6 @@ import ( "flag" "fmt" "net/http" - "os" "os/signal" "syscall" @@ -17,6 +16,7 @@ import ( "github.com/joho/godotenv" "go.uber.org/zap" + "github.com/ozontech/seq-ui/buildinfo" "github.com/ozontech/seq-ui/internal/api" admin_v1 "github.com/ozontech/seq-ui/internal/api/admin/v1" dashboards_v1 "github.com/ozontech/seq-ui/internal/api/dashboards/v1" @@ -39,75 +39,61 @@ import ( "github.com/ozontech/seq-ui/tracing" ) -const ( - defaultConfig = "config/config.example.yaml" +const defaultConfig = "config/config.example.yaml" - defaultClientMaxRecvMsgSize = 4 * 1024 * 1024 // 4MB -) - -var ( - configPath = flag.String("config", defaultConfig, "application config") -) - -func init() { +func main() { // Load .env file for local development (optional). // OS environment variables take precedence. _ = godotenv.Load() -} -func main() { - ctx, cancel := signal.NotifyContext(context.Background(), - syscall.SIGTERM, - os.Interrupt, + configPath := flag.String("config", defaultConfig, "application config") + flag.Parse() + + logger.Info("hi, I am seq-ui", + zap.String("version", buildinfo.Version), + zap.String("build_time", buildinfo.BuildTime), ) + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() - run(ctx) + run(ctx, *configPath) } -func run(ctx context.Context) { - flag.Parse() - - if *configPath == defaultConfig { +func run(ctx context.Context, configPath string) { + if configPath == defaultConfig { logger.Warn("app uses the default config file, to provide your own config use -config flag") } - cfg, err := loader.FromFile(*configPath) + cfg, err := loader.FromFile(configPath) if err != nil { - logger.Fatal("read config file error", zap.Error(err)) + logger.Fatal("failed to read config", zap.Error(err)) } - if tracingCfg, err := tracing.Initialize(); err != nil { - logger.Error("tracing initialization failed", zap.Error(err)) - } else { - logger.Info( - "tracing initialization success", - zap.String("service_name", tracingCfg.ServiceName), - zap.String("agent_host", tracingCfg.AgentHost), - zap.String("agent_port", tracingCfg.AgentPort), - zap.Float64("sampler_param", tracingCfg.SamplerParam)) - } - - logger.Info("initializing caches") - if err := cache.Init(ctx, &cfg.Cache); err != nil { - logger.Fatal("failed to init caches", zap.Error(err)) + if err := tracing.Init(); err != nil { + logger.Error("failed to init tracing", zap.Error(err)) } registrar := initApp(ctx, cfg) serv, err := server.New(ctx, cfg, registrar) if err != nil { - logger.Fatal("app init error", zap.Error(err)) + logger.Fatal("failed to init servers", zap.Error(err)) } // Run launches configured grpc/http servers and debug server. On successful - // http.ErrServerClosed is returned because of http.Server.Serve. + // shutdown http.ErrServerClosed is returned because of http.Server.Serve. if err = serv.Run(ctx); !errors.Is(err, http.ErrServerClosed) { - logger.Fatal("app run", zap.Error(err)) + logger.Fatal("failed to run servers", zap.Error(err)) } } func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { + logger.Info("initializing caches") + if err := cache.Init(ctx, &cfg.Cache); err != nil { + logger.Fatal("failed to init caches", zap.Error(err)) + } + logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { @@ -126,7 +112,6 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { logger.Fatal("failed to init db", zap.Error(err)) } - // Это ужас пока оставим, на создание handlers не смотри пока // var massExportV1 *massexport_v1.MassExport // if cfg.Handlers.MassExport != nil { // logger.Info("initializing mass export") @@ -147,7 +132,7 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { repo := repository.New(db, cfg.DB.RequestTimeout) userProfilesSvc := userprofile.New(repo.UserProfiles, repo.FavoriteQueries) dashboardsSvc := dashboards.New(repo.Dashboards) - profiles.InitProfiles(repo.UserProfiles.GetOrCreate) + profiles.Init(repo.UserProfiles.GetOrCreate) logger.Info("initializing user profile") userProfileV1 = userprofile_v1.New(userProfilesSvc) @@ -188,7 +173,7 @@ func initSeqDBClients(ctx context.Context, cfg *config.Config) (map[string]seqdb for _, clientCfg := range cfg.Clients.SeqDB { client, err := createSeqBDClient(ctx, &clientCfg, cfg.Handlers.SeqAPI) if err != nil { - return nil, fmt.Errorf("failed to create seq_db client %q: %w", clientCfg.ID, err) + return nil, fmt.Errorf("failed to create seq-db client %q: %w", clientCfg.ID, err) } clients[clientCfg.ID] = client } @@ -211,14 +196,11 @@ func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *con } } - clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * int(maxSearchLimit) - clientMaxRecvMsgSize = max(clientMaxRecvMsgSize, defaultClientMaxRecvMsgSize) - clientParams := seqdb.ClientParams{ Addrs: cfg.Addrs, Timeout: cfg.Timeout, MaxRetries: cfg.RequestRetries, - MaxRecvMsgSize: clientMaxRecvMsgSize, + MaxRecvMsgSize: cfg.AvgDocSize * 1024 * int(maxSearchLimit), } if cfg.RequestRetriesBackoff != nil { clientParams.InitialRetryBackoff = cfg.RequestRetriesBackoff.InitialInterval @@ -242,7 +224,7 @@ func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { } pgxCfg, err := pgxpool.ParseConfig(cfg.ConnString()) if err != nil { - return nil, fmt.Errorf("can't parse connection string: %w", err) + return nil, fmt.Errorf("failed to parse connection string: %w", err) } if !*cfg.UsePreparedStatements { // By default, pgx uses the QueryExecModeCacheStatement and automatically prepares and caches prepared statements. @@ -253,7 +235,11 @@ func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { pool, err := pgxpool.NewWithConfig(ctx, pgxCfg) if err != nil { - return nil, fmt.Errorf("can't create pgx pool: %w", err) + return nil, fmt.Errorf("failed to create pgx pool: %w", err) + } + + if err := pool.Ping(ctx); err != nil { + return nil, fmt.Errorf("failed to ping db: %w", err) } return pool, nil diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index dceccc1..22150ee 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -28,7 +28,7 @@ func (m *v1tov2) migrate() v2.Config { dst.Cache = m.migrateCache(v1Cache) dst.Clients = m.migrateClients(m.src) dst.DB = m.migrateDB(m.src.Server) - dst.Handlers = m.migrateHandlers(m.src.Handlers, &dst, v1Cache) + dst.Handlers = m.migrateHandlers(m.src.Handlers, v1Cache) return dst } @@ -293,14 +293,11 @@ func (m *v1tov2) migrateHandlers(src *v1.Handlers, v1Cache v1.Cache) v2.Handlers dst := v2.Handlers{ SeqAPI: m.migrateSeqAPI(&src.SeqAPI, v1Cache), AsyncSearch: m.migrateAsyncSearch(&src.AsyncSearch, &src.SeqAPI), - // ErrorGroups: m.migrateErrorGroups(src.ErrorGroups), - // Admin: m.migrateAdmin(src.Admin, v1Cache), + ErrorGroups: m.migrateErrorGroups(&src.ErrorGroups, m.src.Server), + Admin: m.migrateAdmin(src.Admin, v1Cache), + // MassExport: m.migrateMassExport(src.MassExport, cfg), } - // if src.MassExport != nil { - // dst.MassExport = m.migrateMassExport(src.MassExport, cfg) - // } - return dst } @@ -439,23 +436,23 @@ func (m *v1tov2) migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet return dst } -// func (m *v1tov2) migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { -// if m.isEgEmpty(eg) { -// return nil -// } -// -// return &v2.ErrorGroups{ -// CHID: v2.DefaultCHClientID, -// Options: v2.ErrorGroupsOptions{ -// LogTagsMapping: v2.LogTagsMapping{ -// Env: eg.LogTagsMapping.Env, -// Service: eg.LogTagsMapping.Service, -// Release: eg.LogTagsMapping.Release, -// }, -// QueryFilter: eg.QueryFilter, -// }, -// } -// } +func (m *v1tov2) migrateErrorGroups(eg *v1.ErrorGroups, server *v1.Server) *v2.ErrorGroups { + if server == nil || server.CH == nil { + return nil + } + + return &v2.ErrorGroups{ + Options: v2.ErrorGroupsOptions{ + LogTagsMapping: v2.LogTagsMapping{ + Env: eg.LogTagsMapping.Env, + Service: eg.LogTagsMapping.Service, + Release: eg.LogTagsMapping.Release, + }, + QueryFilter: eg.QueryFilter, + }, + } +} + func (m *v1tov2) migrateAsyncSearch(src *v1.AsyncSearch, seqAPI *v1.SeqAPI) *v2.AsyncSearch { dst := &v2.AsyncSearch{ Options: v2.AsyncSearchOptions{ @@ -475,49 +472,53 @@ func (m *v1tov2) migrateAsyncSearch(src *v1.AsyncSearch, seqAPI *v1.SeqAPI) *v2. return dst } -// func (m *v1tov2) migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { -// dst := &v2.MassExport{ -// SeqDBID: v2.DefaultSeqDBClientID, -// Options: v2.MassExportOptions{ -// BatchSize: me.BatchSize, -// WorkersCount: me.WorkersCount, -// TasksChannelSize: me.TasksChannelSize, -// PartLength: me.PartLength, -// URLPrefix: me.URLPrefix, -// AllowedUsers: me.AllowedUsers, -// FileStore: m.migrateFileStore(me.FileStore), -// DownloadParams: m.migrateDownloadParams(me.SeqProxyDownloader), -// }, -// } +// func (m *v1tov2) migrateMassExport(src *v1.MassExport, cfg *v2.Config) *v2.MassExport { +// if src == nil { +// return nil +// } // -// if me.SessionStore != nil { -// cfg.Cache.Redis = append(cfg.Cache.Redis, m.migrateRedis(&me.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) +// dst := &v2.MassExport{ +// SeqDBID: v2.DefaultSeqDBClientID, +// Options: v2.MassExportOptions{ +// BatchSize: src.BatchSize, +// WorkersCount: src.WorkersCount, +// TasksChannelSize: src.TasksChannelSize, +// PartLength: src.PartLength, +// URLPrefix: src.URLPrefix, +// AllowedUsers: src.AllowedUsers, +// FileStore: m.migrateFileStore(src.FileStore), +// DownloadParams: m.migrateDownloadParams(m.src.SeqProxyDownloader), +// }, +// } // -// dst.Options.SessionStore = v2.SessionStore{ -// RedisID: v2.DefaultMassExportRedisID, -// ExportLifetime: me.SessionStore.ExportLifetime, -// } -// } +// if me.SessionStore != nil { +// cfg.Cache.Redis = append(cfg.Cache.Redis, m.migrateRedis(&src.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) // -// return dst -// } +// dst.Options.SessionStore = v2.SessionStore{ +// RedisID: v2.DefaultMassExportRedisID, +// ExportLifetime: src.SessionStore.ExportLifetime, +// } +// } // -// func (m *v1tov2) migrateFileStore(fs *v1.FileStore) v2.FileStore { -// if fs == nil || fs.S3 == nil { -// return v2.FileStore{} -// } +// return dst +// } // -// return v2.FileStore{ -// S3: &v2.S3{ -// Endpoint: fs.S3.Endpoint, -// AccessKeyID: fs.S3.AccessKeyID, -// SecretAccessKey: fs.S3.SecretAccessKey, -// BucketName: fs.S3.BucketName, -// EnableSSL: fs.S3.EnableSSl, -// }, -// } -// } +// func (m *v1tov2) migrateFileStore(fs *v1.FileStore) v2.FileStore { +// if fs == nil || fs.S3 == nil { +// return v2.FileStore{} +// } // +// return v2.FileStore{ +// S3: &v2.S3{ +// Endpoint: fs.S3.Endpoint, +// AccessKeyID: fs.S3.AccessKeyID, +// SecretAccessKey: fs.S3.SecretAccessKey, +// BucketName: fs.S3.BucketName, +// EnableSSL: fs.S3.EnableSSl, +// }, +// } +// } + // func (m *v1tov2) migrateDownloadParams(src *v1.SeqProxyDownloader) v2.DownloadParams { // if src == nil { // return v2.DownloadParams{} @@ -529,29 +530,27 @@ func (m *v1tov2) migrateAsyncSearch(src *v1.AsyncSearch, seqAPI *v1.SeqAPI) *v2. // MaxRetryBackoff: src.MaxRetryBackoff, // } // } -// -// func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { -// if src == nil { -// return nil -// } -// -// dst := &v2.Admin{ -// Options: v2.AdminOptions{ -// SuperUsers: src.SuperUsers, -// }, -// } -// -// _, redisID := m.defaultCacheIDs(v1Cache) -// if redisID != "" { -// dst.Options.Cache = &v2.HandlerCache{ -// ID: redisID, -// TTL: src.CacheTTL, -// } -// } -// -// return dst -// } -// +func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { + if src == nil { + return nil + } + + dst := &v2.Admin{ + Options: v2.AdminOptions{ + SuperUsers: src.SuperUsers, + }, + } + + if v1Cache.Redis != nil { + dst.Options.Cache = &v2.HandlerCache{ + ID: v2.DefaultRedisWithoutInmemID, + TTL: src.CacheTTL, + } + } + + return dst +} + // defaultCacheIDs returns cache ids same as in v1. // inmemWithRedisID - inmemory+redis or inmemory (v1 events, oidc) // redisOrInmemID - redis or inmemory (v1 logslifespan) @@ -561,9 +560,3 @@ func (m *v1tov2) defaultCacheIDs(src v1.Cache) (inmemWithRedisID, redisOrInmemID } return v2.DefaultInmemID, v2.DefaultInmemID } - -// -// func (m *v1tov2) isEgEmpty(src v1.ErrorGroups) bool { -// return len(src.LogTagsMapping.Env) == 0 && len(src.LogTagsMapping.Service) == 0 && -// len(src.LogTagsMapping.Release) == 0 && len(src.QueryFilter) == 0 -// } diff --git a/internal/app/config/migrate/v1_v2_test.go b/internal/app/config/migrate/v1_v2_test.go index 83f1c00..906eaec 100644 --- a/internal/app/config/migrate/v1_v2_test.go +++ b/internal/app/config/migrate/v1_v2_test.go @@ -1,3 +1,3 @@ package migrate -// TODO +// TODO: write tests diff --git a/internal/pkg/client/seqdb/grpc_client.go b/internal/pkg/client/seqdb/grpc_client.go index 57396a5..0be1dbb 100644 --- a/internal/pkg/client/seqdb/grpc_client.go +++ b/internal/pkg/client/seqdb/grpc_client.go @@ -24,6 +24,8 @@ import ( "github.com/ozontech/seq-ui/metric" ) +const defaultMaxRecvMsgSize = 4 * 1024 * 1024 // 4MB + type grpcSearchResp interface { GetError() *seqproxyapi.Error } @@ -143,7 +145,7 @@ func NewGRPCClient(ctx context.Context, params ClientParams) (*GRPCClient, error opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions( - grpc.MaxCallRecvMsgSize(params.MaxRecvMsgSize), + grpc.MaxCallRecvMsgSize(max(params.MaxRecvMsgSize, defaultMaxRecvMsgSize)), ), grpc.WithUnaryInterceptor(grpc_mw.ChainUnaryClient(unaryInterceptors...)), grpc.WithStreamInterceptor(grpc_mw.ChainStreamClient(streamInterceptors...)), diff --git a/internal/pkg/repository/admin_repo.go b/internal/pkg/repository/admin_repo.go index 0b6eecf..12b48f3 100644 --- a/internal/pkg/repository/admin_repo.go +++ b/internal/pkg/repository/admin_repo.go @@ -45,9 +45,9 @@ func (r *adminRepository) CreateRole(ctx context.Context, req types.CreateRoleRe func (r *adminRepository) AddUsersToRole(ctx context.Context, req types.AddUsersToRoleRequest) error { metricLabels := []string{"users_roles", "INSERT"} - query, args := `INSERT INTO users_roles (user_id, role_id) - SELECT id, $1 FROM user_profiles WHERE user_name=ANY($2) - ON CONFLICT DO NOTHING`, []any{req.RoleID, req.Usernames} + query, args := `INSERT INTO users_roles (user_id, role_id, env) + SELECT id, $1, $2 FROM user_profiles WHERE user_name=ANY($3) + ON CONFLICT DO NOTHING`, []any{req.RoleID, req.Env, req.Usernames} if _, err := r.pool.exec(ctx, metricLabels, query, args...); err != nil { incErrorMetric(err, metricLabels) @@ -59,8 +59,8 @@ func (r *adminRepository) AddUsersToRole(ctx context.Context, req types.AddUsers func (r *adminRepository) DeleteUsersFromRole(ctx context.Context, req types.DeleteUsersFromRoleRequest) error { metricLabels := []string{"users_roles", "DELETE"} - query, args := `DELETE FROM users_roles WHERE role_id=$1 AND user_id IN ( - SELECT id FROM user_profiles WHERE user_name=ANY($2))`, []any{req.RoleID, req.Usernames} + query, args := `DELETE FROM users_roles WHERE role_id=$1 AND env=$2 AND user_id IN ( + SELECT id FROM user_profiles WHERE user_name=ANY($3))`, []any{req.RoleID, req.Env, req.Usernames} if _, err := r.pool.exec(ctx, metricLabels, query, args...); err != nil { incErrorMetric(err, metricLabels) @@ -100,7 +100,7 @@ func (r *adminRepository) GetRoles(ctx context.Context) ([]types.Role, error) { func (r *adminRepository) GetRole(ctx context.Context, req types.GetRoleRequest) (types.RoleInfo, error) { metricLabels := []string{"user_profiles", "SELECT"} query, args := `SELECT up.user_name FROM user_profiles up JOIN users_roles ur ON up.id = ur.user_id - WHERE ur.role_id=$1 ORDER BY up.user_name`, []any{req.RoleID} + WHERE ur.role_id=$1 AND ur.env=$2 ORDER BY up.user_name`, []any{req.RoleID, req.Env} rows, err := r.pool.query(ctx, metricLabels, query, args...) if err != nil { @@ -162,8 +162,8 @@ func (r *adminRepository) DeleteRole(ctx context.Context, req types.DeleteRoleRe return r.txManager.Do(ctx, func(tx pgx.Tx) error { if req.ReplacementRoleID != nil { metricLabels := []string{"users_roles", "INSERT"} - query, args := `INSERT INTO users_roles (user_id, role_id) - SELECT user_id, $1 FROM users_roles WHERE role_id=$2 ON CONFLICT DO NOTHING`, []any{*req.ReplacementRoleID, req.RoleID} + query, args := `INSERT INTO users_roles (user_id, role_id, env) + SELECT user_id, $1, env FROM users_roles WHERE role_id=$2 ON CONFLICT DO NOTHING`, []any{*req.ReplacementRoleID, req.RoleID} if _, err := r.pool.execTx(ctx, tx, metricLabels, query, args...); err != nil { incErrorMetric(err, metricLabels) @@ -209,7 +209,7 @@ func (r *adminRepository) GetUserPermissions(ctx context.Context, req types.GetU JOIN users_roles ur ON up.id=ur.user_id JOIN roles_permissions rp ON ur.role_id=rp.role_id JOIN permissions p ON rp.permission_id=p.id - WHERE up.user_name=$1 ORDER BY p.value`, []any{req.Username} + WHERE ur.env=$1 up.user_name=$2 ORDER BY p.value`, []any{req.Env, req.Username} rows, err := r.pool.query(ctx, metricLabels, query, args...) if err != nil { diff --git a/internal/pkg/repository/async_searches_repo.go b/internal/pkg/repository/async_searches_repo.go index 4313c74..fcfc99d 100644 --- a/internal/pkg/repository/async_searches_repo.go +++ b/internal/pkg/repository/async_searches_repo.go @@ -125,7 +125,7 @@ func (r *asyncSearchesRepository) DeleteAsyncSearch(ctx context.Context, req typ } func (r *asyncSearchesRepository) DeleteExpiredAsyncSearches(ctx context.Context, req types.DeleteExpiredAsyncSearchesRequest) error { - query, args := "DELETE FROM async_searches WHERE env = $1 expires_at < now()", + query, args := "DELETE FROM async_searches WHERE env = $1 AND expires_at < now()", []any{req.Env} metricLabelsDelete := []string{"async_searches", "DELETE"} diff --git a/migration/12_alter_table_async_searches_add_column_env.sql b/migration/12_alter_table_async_searches_add_column_env.sql deleted file mode 100644 index 52ba0f2..0000000 --- a/migration/12_alter_table_async_searches_add_column_env.sql +++ /dev/null @@ -1,9 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE IF EXISTS async_searches ADD COLUMN IF NOT EXISTS env text NOT NULL DEFAULT ''; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE IF EXISTS async_searches DROP COLUMN IF EXISTS env; --- +goose StatementEnd diff --git a/migration/13_add_column_env.sql b/migration/13_add_column_env.sql new file mode 100644 index 0000000..838482c --- /dev/null +++ b/migration/13_add_column_env.sql @@ -0,0 +1,27 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE IF EXISTS async_searches ADD COLUMN IF NOT EXISTS env text NOT NULL DEFAULT ''; +ALTER TABLE IF EXISTS async_searches DROP CONSTRAINT IF EXISTS async_searches_pkey; +ALTER TABLE IF EXISTS async_searches ADD PRIMARY KEY (env, search_id); + +ALTER TABLE IF EXISTS users_roles ADD COLUMN IF NOT EXISTS env text NOT NULL DEFAULT ''; +ALTER TABLE IF EXISTS users_roles DROP CONSTRAINT IF EXISTS users_roles_pkey; +ALTER TABLE IF EXISTS users_roles ADD PRIMARY KEY (user_id, env, role_id); + +DROP INDEX IF EXISTS idx_users_roles_role_id; +CREATE INDEX IF NOT EXISTS idx_users_roles_role_id_env ON users_roles(role_id, env); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_users_roles_role_id_env; +CREATE INDEX IF NOT EXISTS idx_users_roles_role_id ON users_roles(role_id); + +ALTER TABLE IF EXISTS users_roles DROP CONSTRAINT IF EXISTS users_roles_pkey; +ALTER TABLE IF EXISTS users_roles ADD PRIMARY KEY (user_id, role_id); +ALTER TABLE IF EXISTS users_roles DROP COLUMN IF EXISTS env; + +ALTER TABLE IF EXISTS async_searches DROP CONSTRAINT IF EXISTS async_searches_pkey; +ALTER TABLE IF EXISTS async_searches ADD PRIMARY KEY (search_id); +ALTER TABLE IF EXISTS async_searches DROP COLUMN IF EXISTS env; +-- +goose StatementEnd diff --git a/tracing/tracing.go b/tracing/tracing.go index 43ccf2c..a7d0ccd 100644 --- a/tracing/tracing.go +++ b/tracing/tracing.go @@ -6,6 +6,7 @@ import ( "time" "github.com/caarlos0/env/v11" + "github.com/ozontech/seq-ui/logger" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/jaeger" @@ -14,6 +15,7 @@ import ( tracesdk "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.20.0" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -28,19 +30,19 @@ type Config struct { SamplerParam float64 `env:"TRACING_SAMPLER_PARAM"` } -func Initialize() (*Config, error) { +func Init() error { tracingCfg, err := readConfig() if err != nil { - return nil, fmt.Errorf("failed to read tracing config: %w", err) + return fmt.Errorf("failed to read tracing config: %w", err) } if err := validateTracingConfig(tracingCfg); err != nil { - return nil, err + return err } tp, err := newTracerProvider(tracingCfg) if err != nil { - return nil, fmt.Errorf("can't create tracer provider: %w", err) + return fmt.Errorf("can't create tracer provider: %w", err) } // Register our TracerProvider as the global so any imported @@ -48,7 +50,14 @@ func Initialize() (*Config, error) { otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.TraceContext{}) - return tracingCfg, nil + logger.Info("tracing initialization success", + zap.String("service_name", tracingCfg.ServiceName), + zap.String("agent_host", tracingCfg.AgentHost), + zap.String("agent_port", tracingCfg.AgentPort), + zap.Float64("sampler_param", tracingCfg.SamplerParam), + ) + + return nil } func StartSpan(ctx context.Context, name string) (context.Context, trace.Span) { From ca15195c88ba162dea9ea7481892b7e86c4fb5f8 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Mon, 19 Oct 2026 00:59:23 +0300 Subject: [PATCH 25/29] fix linter --- internal/api/userprofile/v1/grpc/user_profiles_test.go | 2 +- internal/api/userprofile/v1/http/user_profiles_test.go | 2 +- internal/pkg/service/profiles/profiles.go | 2 +- tracing/tracing.go | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/api/userprofile/v1/grpc/user_profiles_test.go b/internal/api/userprofile/v1/grpc/user_profiles_test.go index 5558d95..0ffd540 100644 --- a/internal/api/userprofile/v1/grpc/user_profiles_test.go +++ b/internal/api/userprofile/v1/grpc/user_profiles_test.go @@ -81,7 +81,7 @@ func TestGetUserProfile(t *testing.T) { Times(1) if tt.mockArgs.err == nil { - profiles.InitProfiles(mockedSvc.GetOrCreateUserProfile) + profiles.Init(mockedSvc.GetOrCreateUserProfile) profiles.SetID(tt.mockArgs.resp.UserName, tt.mockArgs.resp.ID) } } diff --git a/internal/api/userprofile/v1/http/user_profiles_test.go b/internal/api/userprofile/v1/http/user_profiles_test.go index 4e0fafe..6c12005 100644 --- a/internal/api/userprofile/v1/http/user_profiles_test.go +++ b/internal/api/userprofile/v1/http/user_profiles_test.go @@ -78,7 +78,7 @@ func TestServeGetUserProfile(t *testing.T) { Times(1) if tt.mockArgs.err == nil { - profiles.InitProfiles(mockedSvc.GetOrCreateUserProfile) + profiles.Init(mockedSvc.GetOrCreateUserProfile) profiles.SetID(tt.mockArgs.resp.UserName, tt.mockArgs.resp.ID) } } diff --git a/internal/pkg/service/profiles/profiles.go b/internal/pkg/service/profiles/profiles.go index a473a97..0b8c815 100644 --- a/internal/pkg/service/profiles/profiles.go +++ b/internal/pkg/service/profiles/profiles.go @@ -11,7 +11,7 @@ type getOrCreateFn func(context.Context, types.GetOrCreateUserProfileRequest) (t var profile *profiles -func InitProfiles(fn getOrCreateFn) { +func Init(fn getOrCreateFn) { profile = &profiles{ idByName: make(map[string]int64), getOrCreateFn: fn, diff --git a/tracing/tracing.go b/tracing/tracing.go index a7d0ccd..b73cb6a 100644 --- a/tracing/tracing.go +++ b/tracing/tracing.go @@ -6,7 +6,6 @@ import ( "time" "github.com/caarlos0/env/v11" - "github.com/ozontech/seq-ui/logger" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/jaeger" @@ -17,6 +16,8 @@ import ( "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/ozontech/seq-ui/logger" ) const ( From 837f498d95e8b6c4cee06d1af238928bd64c6be3 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Tue, 20 Oct 2026 00:50:33 +0300 Subject: [PATCH 26/29] victory --- cmd/config-migrate/main.go | 9 +- cmd/seq-ui/main.go | 80 +- internal/api/admin/v1/admin.go | 6 +- internal/api/admin/v1/grpc/api.go | 15 +- internal/api/admin/v1/grpc/roles.go | 90 +- internal/api/admin/v1/grpc/roles_test.go | 7 - internal/api/admin/v1/http/api.go | 15 +- internal/api/admin/v1/http/roles.go | 85 +- internal/api/admin/v1/http/roles_test.go | 7 - internal/api/errorgroups/v1/errorgroups.go | 6 +- internal/api/errorgroups/v1/grpc/api.go | 4 +- .../errorgroups/v1/grpc/diff_by_releases.go | 2 +- .../api/errorgroups/v1/grpc/get_details.go | 2 +- .../api/errorgroups/v1/grpc/get_groups.go | 2 +- internal/api/errorgroups/v1/grpc/get_hist.go | 2 +- .../api/errorgroups/v1/grpc/get_releases.go | 2 +- .../v1/grpc/get_service_filters.go | 2 +- .../api/errorgroups/v1/grpc/get_services.go | 2 +- .../api/errorgroups/v1/grpc/get_top_groups.go | 2 +- internal/api/errorgroups/v1/http/api.go | 4 +- .../errorgroups/v1/http/diff_by_releases.go | 2 +- .../api/errorgroups/v1/http/get_details.go | 2 +- .../api/errorgroups/v1/http/get_groups.go | 2 +- internal/api/errorgroups/v1/http/get_hist.go | 2 +- .../api/errorgroups/v1/http/get_releases.go | 2 +- .../v1/http/get_service_filters.go | 2 +- .../api/errorgroups/v1/http/get_services.go | 2 +- .../api/errorgroups/v1/http/get_top_groups.go | 2 +- internal/api/massexport/v1/grpc/api.go | 6 +- internal/api/massexport/v1/grpc/cancel.go | 20 +- internal/api/massexport/v1/grpc/check.go | 19 +- internal/api/massexport/v1/grpc/get_all.go | 5 +- internal/api/massexport/v1/grpc/restore.go | 20 +- internal/api/massexport/v1/grpc/start.go | 12 +- internal/api/massexport/v1/http/api.go | 6 +- internal/api/massexport/v1/http/cancel.go | 19 +- internal/api/massexport/v1/http/check.go | 18 +- internal/api/massexport/v1/http/get_all.go | 4 +- internal/api/massexport/v1/http/restore.go | 19 +- internal/api/massexport/v1/http/start.go | 12 +- internal/api/massexport/v1/massexport.go | 6 +- internal/api/seqapi/v1/grpc/aggregation.go | 2 - internal/api/seqapi/v1/grpc/api.go | 85 +- .../api/seqapi/v1/grpc/cancel_async_search.go | 18 +- .../api/seqapi/v1/grpc/delete_async_search.go | 18 +- internal/api/seqapi/v1/grpc/events.go | 23 +- .../v1/grpc/fetch_async_search_result.go | 15 +- internal/api/seqapi/v1/grpc/fields.go | 71 +- .../seqapi/v1/grpc/get_async_searches_list.go | 20 +- internal/api/seqapi/v1/grpc/histogram.go | 2 - internal/api/seqapi/v1/grpc/logs_lifespan.go | 27 +- internal/api/seqapi/v1/grpc/search.go | 13 +- .../api/seqapi/v1/grpc/start_async_search.go | 20 +- internal/api/seqapi/v1/grpc/test_data.go | 24 +- .../seqapi/v1/handler_cache/handler_cache.go | 77 ++ internal/api/seqapi/v1/http/aggregation.go | 2 - internal/api/seqapi/v1/http/aggregation_ts.go | 2 - internal/api/seqapi/v1/http/api.go | 89 +- .../api/seqapi/v1/http/cancel_async_search.go | 19 +- .../api/seqapi/v1/http/delete_async_search.go | 18 +- internal/api/seqapi/v1/http/events.go | 25 +- internal/api/seqapi/v1/http/events_test.go | 1 - internal/api/seqapi/v1/http/export.go | 2 - .../api/seqapi/v1/http/export_async_search.go | 17 +- .../v1/http/fetch_async_search_result.go | 23 +- internal/api/seqapi/v1/http/fields.go | 96 +-- .../seqapi/v1/http/get_async_searches_list.go | 26 +- internal/api/seqapi/v1/http/histogram.go | 2 - internal/api/seqapi/v1/http/logs_lifespan.go | 30 +- internal/api/seqapi/v1/http/search.go | 16 +- .../api/seqapi/v1/http/start_async_search.go | 26 +- internal/api/seqapi/v1/http/test_data.go | 24 +- internal/api/seqapi/v1/seqapi.go | 6 +- internal/app/config/migrate/v1_v2.go | 237 +++--- internal/app/config/v2/config.go | 3 +- internal/app/config/v2/validate.go | 791 ++++++++++-------- internal/app/mw/env.go | 7 +- internal/pkg/cache/redis.go | 41 + internal/pkg/cache/store.go | 18 + internal/pkg/service/admin/mock/service.go | 14 - internal/pkg/service/admin/permissions.go | 2 +- .../pkg/service/admin/permissions_test.go | 9 +- internal/pkg/service/admin/roles_test.go | 21 +- internal/pkg/service/admin/service.go | 43 +- .../pkg/service/async_searches/service.go | 43 +- internal/pkg/service/errorgroups/service.go | 25 +- .../pkg/service/errorgroups/service_test.go | 18 +- .../massexport/seq_proxy_downloader.go | 8 +- internal/pkg/service/massexport/service.go | 51 +- .../service/massexport/sessionstore/redis.go | 257 +++--- 90 files changed, 1521 insertions(+), 1432 deletions(-) create mode 100644 internal/api/seqapi/v1/handler_cache/handler_cache.go create mode 100644 internal/pkg/cache/store.go diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go index 6cc2abc..5ae1e63 100644 --- a/cmd/config-migrate/main.go +++ b/cmd/config-migrate/main.go @@ -13,11 +13,8 @@ import ( "github.com/ozontech/seq-ui/logger" ) -var ( - source = flag.String("source", "", "path to the config file to migrate in place") -) - func main() { + source := flag.String("source", "", "path to the config file to migrate in place") flag.Parse() run(*source) @@ -51,11 +48,11 @@ func run(source string) { if _, err := os.Stat(backupPath); err == nil { logger.Fatal("backup already exists, remove it before re-running", zap.String("backup", backupPath)) } - if err := os.WriteFile(backupPath, sourceCfg, stat.Mode()); err != nil { + if err := os.WriteFile(backupPath, sourceCfg, stat.Mode()); err != nil { //nolint:gosec logger.Fatal("write backup config", zap.String("backup", backupPath), zap.Error(err)) } - if err := os.WriteFile(source, migratedCfg, stat.Mode()); err != nil { + if err := os.WriteFile(source, migratedCfg, stat.Mode()); err != nil { //nolint:gosec logger.Fatal("write migrated config", zap.Error(err)) } diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index e4a4d9a..df4614c 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -21,6 +21,7 @@ import ( admin_v1 "github.com/ozontech/seq-ui/internal/api/admin/v1" dashboards_v1 "github.com/ozontech/seq-ui/internal/api/dashboards/v1" errorgroups_v1 "github.com/ozontech/seq-ui/internal/api/errorgroups/v1" + massexport_v1 "github.com/ozontech/seq-ui/internal/api/massexport/v1" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" "github.com/ozontech/seq-ui/internal/app/config/loader" @@ -33,6 +34,7 @@ import ( asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" "github.com/ozontech/seq-ui/internal/pkg/service/dashboards" "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" + "github.com/ozontech/seq-ui/internal/pkg/service/massexport" "github.com/ozontech/seq-ui/internal/pkg/service/profiles" "github.com/ozontech/seq-ui/internal/pkg/service/userprofile" "github.com/ozontech/seq-ui/logger" @@ -112,21 +114,21 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { logger.Fatal("failed to init db", zap.Error(err)) } - // var massExportV1 *massexport_v1.MassExport - // if cfg.Handlers.MassExport != nil { - // logger.Info("initializing mass export") - // exportServer, err := initExportService(ctx, cfg.Handlers.MassExport, seqDBClients) - // if err != nil { - // logger.Fatal("can't init mass export server", zap.Error(err)) - // } - // massExportV1 = massexport_v1.New(exportServer) - // } + var massExportV1 *massexport_v1.MassExport + if cfg.Handlers.MassExport != nil { + logger.Info("initializing mass export") + massExportSvc, err := massexport.New(ctx, cfg.Handlers.MassExport, seqDBClients) + if err != nil { + logger.Fatal("failed to init mass export", zap.Error(err)) + } + massExportV1 = massexport_v1.New(massExportSvc) + } var ( - adminV1 *admin_v1.Admin - userProfileV1 *userprofile_v1.UserProfile - dashboardsV1 *dashboards_v1.Dashboards - asyncSearchesRepo repository.AsyncSearches + adminV1 *admin_v1.Admin + userProfileV1 *userprofile_v1.UserProfile + dashboardsV1 *dashboards_v1.Dashboards + asyncSearchesSvc map[string]asyncsearches.Service ) if db != nil { repo := repository.New(db, cfg.DB.RequestTimeout) @@ -141,27 +143,23 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { if cfg.Handlers.AsyncSearch != nil { logger.Info("initializing async search") - asyncSearchesRepo = repo.AsyncSearches + asyncSearchesSvc = asyncsearches.New(ctx, cfg.Handlers.AsyncSearch, repo.AsyncSearches, seqDBClients) } if cfg.Handlers.Admin != nil { logger.Info("initializing admin") - adminSvc := admin.NewByEnv(cfg.Handlers.Admin, repo.Admin, repo.UserProfiles) + adminSvc := admin.New(cfg.Handlers.Admin, repo.Admin, repo.UserProfiles) adminV1 = admin_v1.New(adminSvc) } } - var seqApiV1 *seqapi_v1.SeqAPI - if cfg.Handlers.SeqAPI != nil { - logger.Info("initializing seqapi") - asyncSearchesSvc := asyncsearches.NewByEnv(ctx, cfg.Handlers.AsyncSearch, asyncSearchesRepo, seqDBClients) - seqApiV1 = seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, asyncSearchesSvc) - } + logger.Info("initializing seqapi") + seqApiV1 := seqapi_v1.New(&cfg.Handlers.SeqAPI, seqDBClients, asyncSearchesSvc) var errorGroupsV1 *errorgroups_v1.ErrorGroups if cfg.Handlers.ErrorGroups != nil { logger.Info("initializing error groups") - errorGroupsSvc := errorgroups.NewByEnv(cfg.Handlers.ErrorGroups, &cfg.Clients, chClients) + errorGroupsSvc := errorgroups.New(cfg.Handlers.ErrorGroups, &cfg.Clients, chClients) errorGroupsV1 = errorgroups_v1.New(errorGroupsSvc) } @@ -171,7 +169,7 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { func initSeqDBClients(ctx context.Context, cfg *config.Config) (map[string]seqdb.Client, error) { clients := make(map[string]seqdb.Client, len(cfg.Clients.SeqDB)) for _, clientCfg := range cfg.Clients.SeqDB { - client, err := createSeqBDClient(ctx, &clientCfg, cfg.Handlers.SeqAPI) + client, err := createSeqBDClient(ctx, &clientCfg, &cfg.Handlers.SeqAPI) if err != nil { return nil, fmt.Errorf("failed to create seq-db client %q: %w", clientCfg.ID, err) } @@ -183,16 +181,14 @@ func initSeqDBClients(ctx context.Context, cfg *config.Config) (map[string]seqdb func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *config.SeqAPI) (seqdb.Client, error) { var maxSearchLimit int32 - if seqAPI != nil { - if len(seqAPI.Envs) == 0 { - maxSearchLimit = seqAPI.Options.Limits.MaxSearchLimit - } + if len(seqAPI.Envs) == 0 { + maxSearchLimit = seqAPI.Options.Limits.MaxSearchLimit + } - // one seqdb client can be used by several envs, so it must accept the largest one - for _, env := range seqAPI.Envs { - if env.SeqDBID == cfg.ID { - maxSearchLimit = max(maxSearchLimit, env.Options.Limits.MaxSearchLimit) - } + // one seqdb client can be used by several envs, so it must accept the largest one + for _, env := range seqAPI.Envs { + if env.SeqDBID == cfg.ID { + maxSearchLimit = max(maxSearchLimit, env.Options.Limits.MaxSearchLimit) } } @@ -238,29 +234,9 @@ func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { return nil, fmt.Errorf("failed to create pgx pool: %w", err) } - if err := pool.Ping(ctx); err != nil { - return nil, fmt.Errorf("failed to ping db: %w", err) - } - return pool, nil } -// func initExportService(ctx context.Context, cfg config.MassExport, redisCfg *config.Redis, client seqdb.Client) (massexport.Service, error) { -// sessionStore, err := sessionstore.NewRedisSessionStore(ctx, redisCfg, cfg.SessionStore.ExportLifetime) -// if err != nil { -// return nil, fmt.Errorf("init session store: %w", err) -// } -// logger.Info("session store initialized") -// -// fileStore, err := filestore.NewS3(ctx, cfg.FileStore.S3) -// if err != nil { -// return nil, fmt.Errorf("init file store: %w", err) -// } -// logger.Info("file store initialized") -// -// return massexport.NewService(ctx, cfg, sessionStore, fileStore, client) -// } - func initClickHouseClients(ctx context.Context, cfg *config.Config) (map[string]driver.Conn, error) { clients := make(map[string]driver.Conn, len(cfg.Clients.ClickHouse)) for _, clientCfg := range cfg.Clients.ClickHouse { diff --git a/internal/api/admin/v1/admin.go b/internal/api/admin/v1/admin.go index ac8c549..c8c52ab 100644 --- a/internal/api/admin/v1/admin.go +++ b/internal/api/admin/v1/admin.go @@ -13,10 +13,10 @@ type Admin struct { httpAPI *http_api.API } -func New(services map[string]admin.Service) *Admin { +func New(serviceByEnv map[string]admin.Service) *Admin { return &Admin{ - grpcAPI: grpc_api.New(services), - httpAPI: http_api.New(services), + grpcAPI: grpc_api.New(serviceByEnv), + httpAPI: http_api.New(serviceByEnv), } } diff --git a/internal/api/admin/v1/grpc/api.go b/internal/api/admin/v1/grpc/api.go index 12072f2..5fb0463 100644 --- a/internal/api/admin/v1/grpc/api.go +++ b/internal/api/admin/v1/grpc/api.go @@ -9,21 +9,14 @@ import ( type API struct { api.UnimplementedAdminServiceServer - services map[string]admin.Service + serviceByEnv map[string]admin.Service availablePermissions []*api.PermissionGroup } -func New(services map[string]admin.Service) *API { - // available permissions are the same for all envs - var availablePermissions []*api.PermissionGroup - for _, svc := range services { - availablePermissions = availablePermissionsToProto(svc.GetAvailablePermissions()) - break - } - +func New(serviceByEnv map[string]admin.Service) *API { return &API{ - services: services, - availablePermissions: availablePermissions, + serviceByEnv: serviceByEnv, + availablePermissions: availablePermissionsToProto(admin.GetAvailablePermissions()), } } diff --git a/internal/api/admin/v1/grpc/roles.go b/internal/api/admin/v1/grpc/roles.go index 2e9e397..b3eb6ab 100644 --- a/internal/api/admin/v1/grpc/roles.go +++ b/internal/api/admin/v1/grpc/roles.go @@ -18,23 +18,29 @@ func (a *API) CreateRole(ctx context.Context, req *admin.CreateRoleRequest) (*ad ctx, span := tracing.StartSpan(ctx, "admin_v1_create_role") defer span.End() - span.SetAttributes( - attribute.KeyValue{ + env := types.GetEnvKey(ctx) + + attributes := []attribute.KeyValue{ + { Key: "role_name", Value: attribute.StringValue(req.GetName()), }, - attribute.KeyValue{ + { Key: "permissions_count", Value: attribute.IntValue(len(req.GetPermissions())), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) request := types.CreateRoleRequest{ Name: req.Name, Permissions: permissionGroupsToStrings(req.Permissions), } - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] roleID, err := svc.CreateRole(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) @@ -49,18 +55,24 @@ func (a *API) AddUsersToRole(ctx context.Context, req *admin.AddUsersToRoleReque ctx, span := tracing.StartSpan(ctx, "admin_v1_add_users_to_role") defer span.End() - span.SetAttributes( - attribute.KeyValue{ + env := types.GetEnvKey(ctx) + + attributes := []attribute.KeyValue{ + { Key: "role_id", Value: attribute.IntValue(int(req.GetRoleId())), }, - attribute.KeyValue{ + { Key: "users_count", Value: attribute.IntValue(len(req.GetUsernames())), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] if err := svc.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ RoleID: req.RoleId, Usernames: req.Usernames, @@ -75,18 +87,24 @@ func (a *API) DeleteUsersFromRole(ctx context.Context, req *admin.DeleteUsersFro ctx, span := tracing.StartSpan(ctx, "admin_v1_delete_users_from_role") defer span.End() - span.SetAttributes( - attribute.KeyValue{ + env := types.GetEnvKey(ctx) + + attributes := []attribute.KeyValue{ + { Key: "role_id", Value: attribute.IntValue(int(req.GetRoleId())), }, - attribute.KeyValue{ + { Key: "users_count", Value: attribute.IntValue(len(req.GetUsernames())), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] if err := svc.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ RoleID: req.RoleId, Usernames: req.Usernames, @@ -98,7 +116,7 @@ func (a *API) DeleteUsersFromRole(ctx context.Context, req *admin.DeleteUsersFro } func (a *API) GetRoles(ctx context.Context, _ *admin.GetRolesRequest) (*admin.GetRolesResponse, error) { - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[types.GetEnvKey(ctx)] resp, err := svc.GetRoles(ctx) if err != nil { return nil, grpcutil.ProcessError(err) @@ -114,14 +132,20 @@ func (a *API) GetRole(ctx context.Context, req *admin.GetRoleRequest) (*admin.Ge ctx, span := tracing.StartSpan(ctx, "admin_v1_get_role") defer span.End() - span.SetAttributes( - attribute.KeyValue{ + env := types.GetEnvKey(ctx) + + attributes := []attribute.KeyValue{ + { Key: "role_id", Value: attribute.IntValue(int(req.GetId())), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] roleInfo, err := svc.GetRole(ctx, types.GetRoleRequest{ RoleID: req.Id, }) @@ -138,7 +162,9 @@ func (a *API) UpdateRole(ctx context.Context, req *admin.UpdateRoleRequest) (*ad ctx, span := tracing.StartSpan(ctx, "admin_v1_update_role") defer span.End() - spanAttributes := []attribute.KeyValue{ + env := types.GetEnvKey(ctx) + + attributes := []attribute.KeyValue{ { Key: "role_id", Value: attribute.IntValue(int(req.GetId())), @@ -149,14 +175,17 @@ func (a *API) UpdateRole(ctx context.Context, req *admin.UpdateRoleRequest) (*ad }, } if req.Name != nil { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "role_name", Value: attribute.StringValue(req.GetName()), }) } - span.SetAttributes(spanAttributes...) + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] if err := svc.UpdateRole(ctx, types.UpdateRoleRequest{ RoleID: req.Id, Name: req.Name, @@ -172,21 +201,26 @@ func (a *API) DeleteRole(ctx context.Context, req *admin.DeleteRoleRequest) (*ad ctx, span := tracing.StartSpan(ctx, "admin_v1_delete_role") defer span.End() - spanAttributes := []attribute.KeyValue{ + env := types.GetEnvKey(ctx) + + attributes := []attribute.KeyValue{ { Key: "role_id", Value: attribute.IntValue(int(req.GetId())), }, } if req.ReplacementRoleId != nil { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "replacement_role_id", Value: attribute.IntValue(int(req.GetReplacementRoleId())), }) } - span.SetAttributes(spanAttributes...) + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] if err := svc.DeleteRole(ctx, types.DeleteRoleRequest{ RoleID: req.Id, ReplacementRoleID: req.ReplacementRoleId, diff --git a/internal/api/admin/v1/grpc/roles_test.go b/internal/api/admin/v1/grpc/roles_test.go index 5a45f5d..d25ea56 100644 --- a/internal/api/admin/v1/grpc/roles_test.go +++ b/internal/api/admin/v1/grpc/roles_test.go @@ -27,13 +27,6 @@ func setupTestAPI(t *testing.T) (*API, *mock.MockService) { ctrl := gomock.NewController(t) mockedSvc := mock.NewMockService(ctrl) - mockedSvc.EXPECT(). - GetAvailablePermissions(). - Return([]types.PermissionGroup{ - {Group: "roles", Permissions: []string{"create", "read", "update", "delete"}}, - }). - Times(1) - return New(map[string]admin_service.Service{"": mockedSvc}), mockedSvc } diff --git a/internal/api/admin/v1/http/api.go b/internal/api/admin/v1/http/api.go index 823a2f5..b200e2a 100644 --- a/internal/api/admin/v1/http/api.go +++ b/internal/api/admin/v1/http/api.go @@ -8,21 +8,14 @@ import ( ) type API struct { - services map[string]adminservice.Service + serviceByEnv map[string]adminservice.Service availablePermissions []permissionGroup } -func New(services map[string]adminservice.Service) *API { - // available permissions are the same for all envs - var availablePermissions []permissionGroup - for _, svc := range services { - availablePermissions = parseAvailablePermissions(svc.GetAvailablePermissions()) - break - } - +func New(serviceByEnv map[string]adminservice.Service) *API { return &API{ - services: services, - availablePermissions: availablePermissions, + serviceByEnv: serviceByEnv, + availablePermissions: parseAvailablePermissions(adminservice.GetAvailablePermissions()), } } diff --git a/internal/api/admin/v1/http/roles.go b/internal/api/admin/v1/http/roles.go index 467b976..739299b 100644 --- a/internal/api/admin/v1/http/roles.go +++ b/internal/api/admin/v1/http/roles.go @@ -34,6 +34,7 @@ func (a *API) serveCreateRole(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) var httpReq createRoleRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -41,18 +42,22 @@ func (a *API) serveCreateRole(w http.ResponseWriter, r *http.Request) { return } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "role_name", Value: attribute.StringValue(httpReq.Name), }, - attribute.KeyValue{ + { Key: "permissions_count", Value: attribute.IntValue(len(httpReq.Permissions)), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] roleID, err := svc.CreateRole(ctx, types.CreateRoleRequest{ Name: httpReq.Name, Permissions: parsePermissionGroupsToStrings(httpReq.Permissions), @@ -81,6 +86,7 @@ func (a *API) serveAddUsersToRole(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) roleID, err := getRoleID(r) if err != nil { @@ -94,18 +100,22 @@ func (a *API) serveAddUsersToRole(w http.ResponseWriter, r *http.Request) { return } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "role_id", Value: attribute.IntValue(int(roleID)), }, - attribute.KeyValue{ + { Key: "users_count", Value: attribute.IntValue(len(httpReq.Usernames)), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] if err := svc.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ RoleID: roleID, Usernames: httpReq.Usernames, @@ -133,6 +143,7 @@ func (a *API) serveDeleteUsersFromRole(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) roleID, err := getRoleID(r) if err != nil { @@ -146,18 +157,22 @@ func (a *API) serveDeleteUsersFromRole(w http.ResponseWriter, r *http.Request) { return } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "role_id", Value: attribute.IntValue(int(roleID)), }, - attribute.KeyValue{ + { Key: "users_count", Value: attribute.IntValue(len(httpReq.Usernames)), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] if err := svc.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ RoleID: roleID, Usernames: httpReq.Usernames, @@ -184,7 +199,7 @@ func (a *API) serveGetRoles(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[types.GetEnvKey(ctx)] resp, err := svc.GetRoles(ctx) if err != nil { httputil.ProcessError(wr, err) @@ -212,6 +227,7 @@ func (a *API) serveGetRole(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) roleID, err := getRoleID(r) if err != nil { @@ -219,14 +235,18 @@ func (a *API) serveGetRole(w http.ResponseWriter, r *http.Request) { return } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "role_id", Value: attribute.IntValue(int(roleID)), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[env] roleInfo, err := svc.GetRole(ctx, types.GetRoleRequest{ RoleID: roleID, }) @@ -256,6 +276,7 @@ func (a *API) serveUpdateRole(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) roleID, err := getRoleID(r) if err != nil { @@ -269,7 +290,7 @@ func (a *API) serveUpdateRole(w http.ResponseWriter, r *http.Request) { return } - spanAttributes := []attribute.KeyValue{ + attributes := []attribute.KeyValue{ { Key: "role_id", Value: attribute.IntValue(int(roleID)), @@ -280,14 +301,18 @@ func (a *API) serveUpdateRole(w http.ResponseWriter, r *http.Request) { }, } if httpReq.Name != nil { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "role_name", Value: attribute.StringValue(*httpReq.Name), }) } - span.SetAttributes(spanAttributes...) + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } - svc := a.services[types.GetEnvKey(ctx)] + span.SetAttributes(attributes...) + + svc := a.serviceByEnv[env] if err := svc.UpdateRole(ctx, types.UpdateRoleRequest{ RoleID: roleID, Name: httpReq.Name, @@ -316,6 +341,7 @@ func (a *API) serveDeleteRole(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) roleID, err := getRoleID(r) if err != nil { @@ -329,21 +355,24 @@ func (a *API) serveDeleteRole(w http.ResponseWriter, r *http.Request) { return } - spanAttributes := []attribute.KeyValue{ + attributes := []attribute.KeyValue{ { Key: "role_id", Value: attribute.IntValue(int(roleID)), }, } if httpReq.ReplacementRoleID != nil { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "replacement_role_id", Value: attribute.IntValue(int(*httpReq.ReplacementRoleID)), }) } - span.SetAttributes(spanAttributes...) + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - svc := a.services[types.GetEnvKey(ctx)] + svc := a.serviceByEnv[types.GetEnvKey(ctx)] if err := svc.DeleteRole(ctx, types.DeleteRoleRequest{ RoleID: roleID, ReplacementRoleID: httpReq.ReplacementRoleID, diff --git a/internal/api/admin/v1/http/roles_test.go b/internal/api/admin/v1/http/roles_test.go index c4281f8..53002e2 100644 --- a/internal/api/admin/v1/http/roles_test.go +++ b/internal/api/admin/v1/http/roles_test.go @@ -26,13 +26,6 @@ func setupTestAPI(t *testing.T) (*API, *mock.MockService) { ctrl := gomock.NewController(t) mockedSvc := mock.NewMockService(ctrl) - mockedSvc.EXPECT(). - GetAvailablePermissions(). - Return([]types.PermissionGroup{ - {Group: "roles", Permissions: []string{"create", "read", "update", "delete"}}, - }). - Times(1) - return New(map[string]admin_service.Service{"": mockedSvc}), mockedSvc } diff --git a/internal/api/errorgroups/v1/errorgroups.go b/internal/api/errorgroups/v1/errorgroups.go index 97e0d7a..243e9e2 100644 --- a/internal/api/errorgroups/v1/errorgroups.go +++ b/internal/api/errorgroups/v1/errorgroups.go @@ -13,10 +13,10 @@ type ErrorGroups struct { httpAPI *http_api.API } -func New(services map[string]errorgroups.Service) *ErrorGroups { +func New(serviceByEnv map[string]errorgroups.Service) *ErrorGroups { return &ErrorGroups{ - grpcAPI: grpc_api.New(services), - httpAPI: http_api.New(services), + grpcAPI: grpc_api.New(serviceByEnv), + httpAPI: http_api.New(serviceByEnv), } } diff --git a/internal/api/errorgroups/v1/grpc/api.go b/internal/api/errorgroups/v1/grpc/api.go index fb807ac..04b7311 100644 --- a/internal/api/errorgroups/v1/grpc/api.go +++ b/internal/api/errorgroups/v1/grpc/api.go @@ -16,9 +16,9 @@ type API struct { services map[string]errorgroups.Service } -func New(services map[string]errorgroups.Service) *API { +func New(serviceByEnv map[string]errorgroups.Service) *API { return &API{ - services: services, + services: serviceByEnv, } } diff --git a/internal/api/errorgroups/v1/grpc/diff_by_releases.go b/internal/api/errorgroups/v1/grpc/diff_by_releases.go index 7b68c33..379dc66 100644 --- a/internal/api/errorgroups/v1/grpc/diff_by_releases.go +++ b/internal/api/errorgroups/v1/grpc/diff_by_releases.go @@ -26,7 +26,7 @@ func (a *API) DiffByReleases(ctx context.Context, req *errorgroups.DiffByRelease {Key: "order", Value: attribute.StringValue(req.Order.String())}, {Key: "with_total", Value: attribute.BoolValue(req.WithTotal)}, } - if reqEnv != nil { + if reqEnv != nil && *reqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Source != nil { diff --git a/internal/api/errorgroups/v1/grpc/get_details.go b/internal/api/errorgroups/v1/grpc/get_details.go index 091d50c..4602d57 100644 --- a/internal/api/errorgroups/v1/grpc/get_details.go +++ b/internal/api/errorgroups/v1/grpc/get_details.go @@ -28,7 +28,7 @@ func (a *API) GetDetails(ctx context.Context, req *errorgroups.GetDetailsRequest if req.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*req.Service)}) } - if reqEnv != nil { + if reqEnv != nil && *reqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Release != nil { diff --git a/internal/api/errorgroups/v1/grpc/get_groups.go b/internal/api/errorgroups/v1/grpc/get_groups.go index 52baad5..37ac73c 100644 --- a/internal/api/errorgroups/v1/grpc/get_groups.go +++ b/internal/api/errorgroups/v1/grpc/get_groups.go @@ -26,7 +26,7 @@ func (a *API) GetGroups(ctx context.Context, req *errorgroups.GetGroupsRequest) {Key: "order", Value: attribute.StringValue(string(req.Order))}, {Key: "with_total", Value: attribute.BoolValue(req.WithTotal)}, } - if reqEnv != nil { + if reqEnv != nil && *reqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Release != nil { diff --git a/internal/api/errorgroups/v1/grpc/get_hist.go b/internal/api/errorgroups/v1/grpc/get_hist.go index eaf1649..d02a0b8 100644 --- a/internal/api/errorgroups/v1/grpc/get_hist.go +++ b/internal/api/errorgroups/v1/grpc/get_hist.go @@ -29,7 +29,7 @@ func (a *API) GetHist(ctx context.Context, req *errorgroups.GetHistRequest) (*er if req.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*req.Service)}) } - if reqEnv != nil { + if reqEnv != nil && *reqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Release != nil { diff --git a/internal/api/errorgroups/v1/grpc/get_releases.go b/internal/api/errorgroups/v1/grpc/get_releases.go index 7f27655..09c5d9e 100644 --- a/internal/api/errorgroups/v1/grpc/get_releases.go +++ b/internal/api/errorgroups/v1/grpc/get_releases.go @@ -20,7 +20,7 @@ func (a *API) GetReleases(ctx context.Context, req *errorgroups.GetReleasesReque attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, } - if reqEnv != nil { + if reqEnv != nil && *reqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } span.SetAttributes(attributes...) diff --git a/internal/api/errorgroups/v1/grpc/get_service_filters.go b/internal/api/errorgroups/v1/grpc/get_service_filters.go index 102a0b2..62e58ba 100644 --- a/internal/api/errorgroups/v1/grpc/get_service_filters.go +++ b/internal/api/errorgroups/v1/grpc/get_service_filters.go @@ -20,7 +20,7 @@ func (a *API) GetServiceFilters(ctx context.Context, req *errorgroups.GetService attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, } - if reqEnv != nil { + if reqEnv != nil && *reqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } span.SetAttributes(attributes...) diff --git a/internal/api/errorgroups/v1/grpc/get_services.go b/internal/api/errorgroups/v1/grpc/get_services.go index 215eb2e..53d7857 100644 --- a/internal/api/errorgroups/v1/grpc/get_services.go +++ b/internal/api/errorgroups/v1/grpc/get_services.go @@ -20,7 +20,7 @@ func (a *API) GetServices(ctx context.Context, req *errorgroups.GetServicesReque attributes := []attribute.KeyValue{ {Key: "query", Value: attribute.StringValue(req.Query)}, } - if reqEnv != nil { + if reqEnv != nil && *reqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } span.SetAttributes(attributes...) diff --git a/internal/api/errorgroups/v1/grpc/get_top_groups.go b/internal/api/errorgroups/v1/grpc/get_top_groups.go index 475122b..cdec3a5 100644 --- a/internal/api/errorgroups/v1/grpc/get_top_groups.go +++ b/internal/api/errorgroups/v1/grpc/get_top_groups.go @@ -23,7 +23,7 @@ func (a *API) GetTopGroups(ctx context.Context, req *errorgroups.GetTopGroupsReq {Key: "offset", Value: attribute.IntValue(int(req.Offset))}, {Key: "with_total", Value: attribute.BoolValue(req.WithTotal)}, } - if reqEnv != nil { + if reqEnv != nil && *reqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Source != nil { diff --git a/internal/api/errorgroups/v1/http/api.go b/internal/api/errorgroups/v1/http/api.go index 3ca9378..4deb6ea 100644 --- a/internal/api/errorgroups/v1/http/api.go +++ b/internal/api/errorgroups/v1/http/api.go @@ -15,9 +15,9 @@ type API struct { services map[string]errorgroups.Service } -func New(services map[string]errorgroups.Service) *API { +func New(serviceByEnv map[string]errorgroups.Service) *API { return &API{ - services: services, + services: serviceByEnv, } } diff --git a/internal/api/errorgroups/v1/http/diff_by_releases.go b/internal/api/errorgroups/v1/http/diff_by_releases.go index d7f7e0b..237b9e2 100644 --- a/internal/api/errorgroups/v1/http/diff_by_releases.go +++ b/internal/api/errorgroups/v1/http/diff_by_releases.go @@ -46,7 +46,7 @@ func (a *API) serveDiffByReleases(w http.ResponseWriter, r *http.Request) { {Key: "order", Value: attribute.StringValue(string(httpReq.Order))}, {Key: "with_total", Value: attribute.BoolValue(httpReq.WithTotal)}, } - if httpReqEnv != nil { + if httpReqEnv != nil && *httpReqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Source != nil { diff --git a/internal/api/errorgroups/v1/http/get_details.go b/internal/api/errorgroups/v1/http/get_details.go index 3b8759a..9efce0c 100644 --- a/internal/api/errorgroups/v1/http/get_details.go +++ b/internal/api/errorgroups/v1/http/get_details.go @@ -48,7 +48,7 @@ func (a *API) serveGetDetails(w http.ResponseWriter, r *http.Request) { attributes := []attribute.KeyValue{ {Key: "group_hash", Value: attribute.StringValue(httpReq.GroupHash)}, } - if httpReqEnv != nil { + if httpReqEnv != nil && *httpReqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Service != nil { diff --git a/internal/api/errorgroups/v1/http/get_groups.go b/internal/api/errorgroups/v1/http/get_groups.go index 15580d6..00aed5a 100644 --- a/internal/api/errorgroups/v1/http/get_groups.go +++ b/internal/api/errorgroups/v1/http/get_groups.go @@ -45,7 +45,7 @@ func (a *API) serveGetGroups(w http.ResponseWriter, r *http.Request) { {Key: "order", Value: attribute.StringValue(string(httpReq.Order))}, {Key: "with_total", Value: attribute.BoolValue(httpReq.WithTotal)}, } - if httpReqEnv != nil { + if httpReqEnv != nil && *httpReqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Release != nil { diff --git a/internal/api/errorgroups/v1/http/get_hist.go b/internal/api/errorgroups/v1/http/get_hist.go index b3752ca..1677acc 100644 --- a/internal/api/errorgroups/v1/http/get_hist.go +++ b/internal/api/errorgroups/v1/http/get_hist.go @@ -44,7 +44,7 @@ func (a *API) serveGetHist(w http.ResponseWriter, r *http.Request) { if httpReq.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*httpReq.Service)}) } - if httpReqEnv != nil { + if httpReqEnv != nil && *httpReqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Release != nil { diff --git a/internal/api/errorgroups/v1/http/get_releases.go b/internal/api/errorgroups/v1/http/get_releases.go index 2a79f5a..21071c2 100644 --- a/internal/api/errorgroups/v1/http/get_releases.go +++ b/internal/api/errorgroups/v1/http/get_releases.go @@ -39,7 +39,7 @@ func (a *API) serveGetReleases(w http.ResponseWriter, r *http.Request) { attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, } - if httpReqEnv != nil { + if httpReqEnv != nil && *httpReqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } span.SetAttributes(attributes...) diff --git a/internal/api/errorgroups/v1/http/get_service_filters.go b/internal/api/errorgroups/v1/http/get_service_filters.go index 3b27e52..bc2914a 100644 --- a/internal/api/errorgroups/v1/http/get_service_filters.go +++ b/internal/api/errorgroups/v1/http/get_service_filters.go @@ -39,7 +39,7 @@ func (a *API) serveGetServiceFilters(w http.ResponseWriter, r *http.Request) { attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, } - if httpReqEnv != nil { + if httpReqEnv != nil && *httpReqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } span.SetAttributes(attributes...) diff --git a/internal/api/errorgroups/v1/http/get_services.go b/internal/api/errorgroups/v1/http/get_services.go index f630b1f..2d88452 100644 --- a/internal/api/errorgroups/v1/http/get_services.go +++ b/internal/api/errorgroups/v1/http/get_services.go @@ -39,7 +39,7 @@ func (a *API) serveGetServices(w http.ResponseWriter, r *http.Request) { attributes := []attribute.KeyValue{ {Key: "query", Value: attribute.StringValue(httpReq.Query)}, } - if httpReqEnv != nil { + if httpReqEnv != nil && *httpReqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } span.SetAttributes(attributes...) diff --git a/internal/api/errorgroups/v1/http/get_top_groups.go b/internal/api/errorgroups/v1/http/get_top_groups.go index ea009e6..460a8c8 100644 --- a/internal/api/errorgroups/v1/http/get_top_groups.go +++ b/internal/api/errorgroups/v1/http/get_top_groups.go @@ -42,7 +42,7 @@ func (a *API) serveGetTopGroups(w http.ResponseWriter, r *http.Request) { {Key: "offset", Value: attribute.IntValue(int(httpReq.Offset))}, {Key: "with_total", Value: attribute.BoolValue(httpReq.WithTotal)}, } - if httpReqEnv != nil { + if httpReqEnv != nil && *httpReqEnv != "" { attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Source != nil { diff --git a/internal/api/massexport/v1/grpc/api.go b/internal/api/massexport/v1/grpc/api.go index bb0b2c5..b1a40f0 100644 --- a/internal/api/massexport/v1/grpc/api.go +++ b/internal/api/massexport/v1/grpc/api.go @@ -8,11 +8,11 @@ import ( type API struct { api.UnimplementedMassExportServiceServer - exporter massexport.Service + serviceByEnv map[string]massexport.Service } -func New(exporter massexport.Service) *API { +func New(serviceByEnv map[string]massexport.Service) *API { return &API{ - exporter: exporter, + serviceByEnv: serviceByEnv, } } diff --git a/internal/api/massexport/v1/grpc/cancel.go b/internal/api/massexport/v1/grpc/cancel.go index e575c6c..1977a83 100644 --- a/internal/api/massexport/v1/grpc/cancel.go +++ b/internal/api/massexport/v1/grpc/cancel.go @@ -6,6 +6,7 @@ import ( "go.opentelemetry.io/otel/attribute" "github.com/ozontech/seq-ui/internal/api/grpcutil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/massexport/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -14,12 +15,21 @@ func (a *API) Cancel(ctx context.Context, req *massexport.CancelRequest) (*masse ctx, span := tracing.StartSpan(ctx, "massexport_v1_cancel") defer span.End() - span.SetAttributes(attribute.KeyValue{ - Key: "session_id", - Value: attribute.StringValue(req.GetSessionId()), - }) + env := types.GetEnvKey(ctx) + svc := a.serviceByEnv[env] - err := a.exporter.CancelExport(ctx, req.GetSessionId()) + attributes := []attribute.KeyValue{ + { + Key: "session_id", + Value: attribute.StringValue(req.GetSessionId()), + }, + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) + + err := svc.CancelExport(ctx, req.GetSessionId()) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/massexport/v1/grpc/check.go b/internal/api/massexport/v1/grpc/check.go index 86133cf..78df2fb 100644 --- a/internal/api/massexport/v1/grpc/check.go +++ b/internal/api/massexport/v1/grpc/check.go @@ -19,12 +19,21 @@ func (a *API) Check(ctx context.Context, req *massexport_v1.CheckRequest) (*mass ctx, span := tracing.StartSpan(ctx, "massexport_v1_check") defer span.End() - span.SetAttributes(attribute.KeyValue{ - Key: "session_id", - Value: attribute.StringValue(req.GetSessionId()), - }) + env := types.GetEnvKey(ctx) + svc := a.serviceByEnv[env] - info, err := a.exporter.CheckExport(ctx, req.GetSessionId()) + attributes := []attribute.KeyValue{ + { + Key: "session_id", + Value: attribute.StringValue(req.GetSessionId()), + }, + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) + + info, err := svc.CheckExport(ctx, req.GetSessionId()) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/massexport/v1/grpc/get_all.go b/internal/api/massexport/v1/grpc/get_all.go index 996b1a8..dde6165 100644 --- a/internal/api/massexport/v1/grpc/get_all.go +++ b/internal/api/massexport/v1/grpc/get_all.go @@ -4,6 +4,7 @@ import ( "context" "github.com/ozontech/seq-ui/internal/api/grpcutil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/massexport/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -12,7 +13,9 @@ func (a *API) GetAll(ctx context.Context, req *massexport.GetAllRequest) (*masse ctx, span := tracing.StartSpan(ctx, "massexport_v1_jobs") defer span.End() - exports, err := a.exporter.GetAll(ctx) + svc := a.serviceByEnv[types.GetEnvKey(ctx)] + + exports, err := svc.GetAll(ctx) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/massexport/v1/grpc/restore.go b/internal/api/massexport/v1/grpc/restore.go index d5eb5ec..e868565 100644 --- a/internal/api/massexport/v1/grpc/restore.go +++ b/internal/api/massexport/v1/grpc/restore.go @@ -7,6 +7,7 @@ import ( "go.opentelemetry.io/otel/attribute" "github.com/ozontech/seq-ui/internal/api/grpcutil" + "github.com/ozontech/seq-ui/internal/app/types" massexport_v1 "github.com/ozontech/seq-ui/pkg/massexport/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -15,12 +16,21 @@ func (a *API) Restore(ctx context.Context, req *massexport_v1.RestoreRequest) (* ctx, span := tracing.StartSpan(ctx, "massexport_v1_restore") defer span.End() - span.SetAttributes(attribute.KeyValue{ - Key: "session_id", - Value: attribute.StringValue(req.GetSessionId()), - }) + env := types.GetEnvKey(ctx) + svc := a.serviceByEnv[env] - err := a.exporter.RestoreExport(ctx, req.GetSessionId()) + attributes := []attribute.KeyValue{ + { + Key: "session_id", + Value: attribute.StringValue(req.GetSessionId()), + }, + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) + + err := svc.RestoreExport(ctx, req.GetSessionId()) if err != nil { return nil, grpcutil.ProcessError(fmt.Errorf("restore export: %w", err)) } diff --git a/internal/api/massexport/v1/grpc/start.go b/internal/api/massexport/v1/grpc/start.go index 41faf2c..1f30b1b 100644 --- a/internal/api/massexport/v1/grpc/start.go +++ b/internal/api/massexport/v1/grpc/start.go @@ -17,10 +17,12 @@ func (a *API) Start(ctx context.Context, req *massexport_v1.StartRequest) (*mass ctx, span := tracing.StartSpan(ctx, "massexport_v1_start") defer span.End() + env := types.GetEnvKey(ctx) + svc := a.serviceByEnv[env] from := req.GetFrom().AsTime() to := req.GetTo().AsTime() - span.SetAttributes([]attribute.KeyValue{ + attributes := []attribute.KeyValue{ { Key: "query", Value: attribute.StringValue(req.GetQuery()), @@ -41,7 +43,11 @@ func (a *API) Start(ctx context.Context, req *massexport_v1.StartRequest) (*mass Key: "name", Value: attribute.StringValue(req.GetName()), }, - }...) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) if req.GetName() == "" { return nil, grpcutil.ProcessError(types.NewErrInvalidRequestField("empty export name")) @@ -60,7 +66,7 @@ func (a *API) Start(ctx context.Context, req *massexport_v1.StartRequest) (*mass return nil, grpcutil.ProcessError(types.NewErrInvalidRequestField("'window' is larger than whole interval")) } - result, err := a.exporter.StartExport(ctx, types.StartExportRequest{ + result, err := svc.StartExport(ctx, types.StartExportRequest{ Query: req.GetQuery(), From: from, To: to, diff --git a/internal/api/massexport/v1/http/api.go b/internal/api/massexport/v1/http/api.go index 9143dc2..89a2660 100644 --- a/internal/api/massexport/v1/http/api.go +++ b/internal/api/massexport/v1/http/api.go @@ -7,12 +7,12 @@ import ( ) type API struct { - exporter massexport.Service + serviceByEnv map[string]massexport.Service } -func New(exporter massexport.Service) *API { +func New(serviceByEnv map[string]massexport.Service) *API { return &API{ - exporter: exporter, + serviceByEnv: serviceByEnv, } } diff --git a/internal/api/massexport/v1/http/cancel.go b/internal/api/massexport/v1/http/cancel.go index 591851f..ea8adbc 100644 --- a/internal/api/massexport/v1/http/cancel.go +++ b/internal/api/massexport/v1/http/cancel.go @@ -8,6 +8,7 @@ import ( "go.opentelemetry.io/otel/attribute" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/tracing" ) @@ -25,6 +26,8 @@ func (a *API) serveCancel(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) + svc := a.serviceByEnv[env] var httpReq cancelRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -32,12 +35,18 @@ func (a *API) serveCancel(w http.ResponseWriter, r *http.Request) { return } - span.SetAttributes(attribute.KeyValue{ - Key: "session_id", - Value: attribute.StringValue(httpReq.SessionID), - }) + attributes := []attribute.KeyValue{ + { + Key: "session_id", + Value: attribute.StringValue(httpReq.SessionID), + }, + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - err := a.exporter.CancelExport(ctx, httpReq.SessionID) + err := svc.CancelExport(ctx, httpReq.SessionID) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/massexport/v1/http/check.go b/internal/api/massexport/v1/http/check.go index 0dde79a..1272f67 100644 --- a/internal/api/massexport/v1/http/check.go +++ b/internal/api/massexport/v1/http/check.go @@ -27,6 +27,8 @@ func (a *API) serveCheck(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) + svc := a.serviceByEnv[env] var httpReq checkRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -34,12 +36,18 @@ func (a *API) serveCheck(w http.ResponseWriter, r *http.Request) { return } - span.SetAttributes(attribute.KeyValue{ - Key: "session_id", - Value: attribute.StringValue(httpReq.SessionID), - }) + attributes := []attribute.KeyValue{ + { + Key: "session_id", + Value: attribute.StringValue(httpReq.SessionID), + }, + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - info, err := a.exporter.CheckExport(ctx, httpReq.SessionID) + info, err := svc.CheckExport(ctx, httpReq.SessionID) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/massexport/v1/http/get_all.go b/internal/api/massexport/v1/http/get_all.go index 92cc24f..94ee573 100644 --- a/internal/api/massexport/v1/http/get_all.go +++ b/internal/api/massexport/v1/http/get_all.go @@ -4,6 +4,7 @@ import ( "net/http" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/tracing" ) @@ -20,8 +21,9 @@ func (a *API) serveJobs(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + svc := a.serviceByEnv[types.GetEnvKey(ctx)] - exports, err := a.exporter.GetAll(ctx) + exports, err := svc.GetAll(ctx) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/massexport/v1/http/restore.go b/internal/api/massexport/v1/http/restore.go index 32b7481..551b79a 100644 --- a/internal/api/massexport/v1/http/restore.go +++ b/internal/api/massexport/v1/http/restore.go @@ -8,6 +8,7 @@ import ( "go.opentelemetry.io/otel/attribute" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/tracing" ) @@ -25,6 +26,8 @@ func (a *API) serveRestore(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) + svc := a.serviceByEnv[env] var httpReq restoreRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -32,12 +35,18 @@ func (a *API) serveRestore(w http.ResponseWriter, r *http.Request) { return } - span.SetAttributes(attribute.KeyValue{ - Key: "session_id", - Value: attribute.StringValue(httpReq.SessionID), - }) + attributes := []attribute.KeyValue{ + { + Key: "session_id", + Value: attribute.StringValue(httpReq.SessionID), + }, + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - err := a.exporter.RestoreExport(ctx, httpReq.SessionID) + err := svc.RestoreExport(ctx, httpReq.SessionID) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/massexport/v1/http/start.go b/internal/api/massexport/v1/http/start.go index c056252..fc4e6a3 100644 --- a/internal/api/massexport/v1/http/start.go +++ b/internal/api/massexport/v1/http/start.go @@ -29,6 +29,8 @@ func (a *API) serveStart(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) + svc := a.serviceByEnv[env] var httpReq startRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -36,7 +38,7 @@ func (a *API) serveStart(w http.ResponseWriter, r *http.Request) { return } - span.SetAttributes([]attribute.KeyValue{ + attributes := []attribute.KeyValue{ { Key: "query", Value: attribute.StringValue(httpReq.Query), @@ -57,7 +59,11 @@ func (a *API) serveStart(w http.ResponseWriter, r *http.Request) { Key: "name", Value: attribute.StringValue(httpReq.Name), }, - }...) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) if httpReq.Name == "" { wr.Error(errors.New("empty export name"), http.StatusBadRequest) @@ -80,7 +86,7 @@ func (a *API) serveStart(w http.ResponseWriter, r *http.Request) { return } - result, err := a.exporter.StartExport(ctx, types.StartExportRequest{ + result, err := svc.StartExport(ctx, types.StartExportRequest{ Query: httpReq.Query, From: httpReq.From, To: httpReq.To, diff --git a/internal/api/massexport/v1/massexport.go b/internal/api/massexport/v1/massexport.go index 78d378d..a441224 100644 --- a/internal/api/massexport/v1/massexport.go +++ b/internal/api/massexport/v1/massexport.go @@ -13,10 +13,10 @@ type MassExport struct { httpAPI *http_api.API } -func New(service massexport.Service) *MassExport { +func New(serviceByEnv map[string]massexport.Service) *MassExport { return &MassExport{ - grpcAPI: grpc_api.New(service), - httpAPI: http_api.New(service), + grpcAPI: grpc_api.New(serviceByEnv), + httpAPI: http_api.New(serviceByEnv), } } diff --git a/internal/api/seqapi/v1/grpc/aggregation.go b/internal/api/seqapi/v1/grpc/aggregation.go index 53c8101..640b5ef 100644 --- a/internal/api/seqapi/v1/grpc/aggregation.go +++ b/internal/api/seqapi/v1/grpc/aggregation.go @@ -48,11 +48,9 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ Value: attribute.IntValue(int(req.GetDownsample())), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) params, err := a.GetParams(env) diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index 080dc7d..d878839 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -2,15 +2,17 @@ package grpc import ( "fmt" + "maps" + "slices" "sort" "strconv" "time" "go.uber.org/zap" + handlercache "github.com/ozontech/seq-ui/internal/api/seqapi/v1/handler_cache" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/mask" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" @@ -19,49 +21,41 @@ import ( ) type apiParams struct { - client seqdb.Client - options *config.SeqAPIOptions - masker *mask.Masker - pinnedFields []*seqapi.Field - systemFields []*seqapi.Field - eventsCache cache.Cache - logsLifespanCache cache.Cache - fieldsCache *fieldsCache - eventsCacheKeyPrefix string - fieldsCacheKey string - logsLifespanCacheKey string + client seqdb.Client + options *config.SeqAPIOptions + masker *mask.Masker + pinnedFields []*seqapi.Field + systemFields []*seqapi.Field + caches handlercache.Caches } type API struct { seqapi.UnimplementedSeqAPIServiceServer - config *config.SeqAPI - paramsByEnv map[string]apiParams - nowFn func() time.Time - asyncSearches map[string]asyncsearches.Service - envsResponse *seqapi.GetEnvsResponse + config *config.SeqAPI + paramsByEnv map[string]apiParams + nowFn func() time.Time + asyncSearchesByEnv map[string]asyncsearches.Service + envsResponse *seqapi.GetEnvsResponse } -func New(cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, asyncSearches map[string]asyncsearches.Service) *API { +func New(cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, asyncSearchesByEnv map[string]asyncsearches.Service) *API { paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { for envName, envConfig := range cfg.Envs { paramsByEnv[envName] = newAPIParams(envName, seqDBСlients[envConfig.SeqDBID], envConfig.Options) } } else { - var client seqdb.Client - for _, c := range seqDBСlients { - client = c - } + client := slices.Collect(maps.Values(seqDBСlients))[0] paramsByEnv[""] = newAPIParams("", client, &cfg.Options) } return &API{ - config: cfg, - paramsByEnv: paramsByEnv, - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + config: cfg, + paramsByEnv: paramsByEnv, + nowFn: time.Now, + asyncSearchesByEnv: asyncSearchesByEnv, + envsResponse: parseEnvs(cfg), } } @@ -71,38 +65,13 @@ func newAPIParams(env string, client seqdb.Client, options *config.SeqAPIOptions logger.Fatal("failed to init masking", zap.String("env", env), zap.Error(err)) } - var eventsCache, logsLifespanCache cache.Cache - if options.Caches.Events != nil { - eventsCache, _ = cache.ByID(options.Caches.Events.ID) - } - if options.Caches.LogsLifespan != nil { - logsLifespanCache, _ = cache.ByID(options.Caches.LogsLifespan.ID) - } - - var fc *fieldsCache - if options.Caches.Fields != nil { - if c, ok := cache.ByID(options.Caches.Fields.ID); ok { - fc = newFieldsCache(c) - } - } - - cacheKeyPrefix := "seqapi_" - if env != "" { - cacheKeyPrefix += env + "_" - } - return apiParams{ - client: client, - options: options, - masker: masker, - pinnedFields: parseFields(options.PinnedFields), - systemFields: parseFields(options.SystemFields), - eventsCache: eventsCache, - logsLifespanCache: logsLifespanCache, - fieldsCache: fc, - eventsCacheKeyPrefix: cacheKeyPrefix + "event_", - fieldsCacheKey: cacheKeyPrefix + "fields", - logsLifespanCacheKey: cacheKeyPrefix + "logs_lifespan", + client: client, + options: options, + masker: masker, + pinnedFields: parseFields(options.PinnedFields), + systemFields: parseFields(options.SystemFields), + caches: handlercache.New(env, options.Caches), } } diff --git a/internal/api/seqapi/v1/grpc/cancel_async_search.go b/internal/api/seqapi/v1/grpc/cancel_async_search.go index 00f2838..4436c69 100644 --- a/internal/api/seqapi/v1/grpc/cancel_async_search.go +++ b/internal/api/seqapi/v1/grpc/cancel_async_search.go @@ -15,26 +15,32 @@ import ( ) func (a *API) CancelAsyncSearch(ctx context.Context, req *seqapi.CancelAsyncSearchRequest) (*seqapi.CancelAsyncSearchResponse, error) { - asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] - if !ok { + if a.asyncSearchesByEnv == nil { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } ctx, span := tracing.StartSpan(ctx, "seqapi_v1_cancel_async_search") defer span.End() + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + if _, err := uuid.FromString(req.SearchId); err != nil { return nil, status.Error(codes.InvalidArgument, "invalid search_id") } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "search_id", Value: attribute.StringValue(req.SearchId), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - resp, err := asyncSearches.CancelAsyncSearch(ctx, req) + resp, err := svc.CancelAsyncSearch(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/delete_async_search.go b/internal/api/seqapi/v1/grpc/delete_async_search.go index 63f2fed..1fa15a2 100644 --- a/internal/api/seqapi/v1/grpc/delete_async_search.go +++ b/internal/api/seqapi/v1/grpc/delete_async_search.go @@ -15,26 +15,32 @@ import ( ) func (a *API) DeleteAsyncSearch(ctx context.Context, req *seqapi.DeleteAsyncSearchRequest) (*seqapi.DeleteAsyncSearchResponse, error) { - asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] - if !ok { + if a.asyncSearchesByEnv == nil { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } ctx, span := tracing.StartSpan(ctx, "seqapi_v1_delete_async_search") defer span.End() + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + if _, err := uuid.FromString(req.SearchId); err != nil { return nil, status.Error(codes.InvalidArgument, "invalid search_id") } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "search_id", Value: attribute.StringValue(req.SearchId), }, - ) + } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - resp, err := asyncSearches.DeleteAsyncSearch(ctx, req) + resp, err := svc.DeleteAsyncSearch(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/events.go b/internal/api/seqapi/v1/grpc/events.go index a604a95..ca892c4 100644 --- a/internal/api/seqapi/v1/grpc/events.go +++ b/internal/api/seqapi/v1/grpc/events.go @@ -27,11 +27,9 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap Value: attribute.StringValue(req.GetId()), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) params, err := a.GetParams(env) @@ -39,18 +37,15 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap return nil, status.Error(codes.InvalidArgument, err.Error()) } - cacheKey := params.eventsCacheKeyPrefix + req.Id - if params.eventsCache != nil { - if cached, err := params.eventsCache.Get(ctx, cacheKey); err == nil { - event := &seqapi.Event{} - if err = proto.Unmarshal([]byte(cached), event); err == nil { - if params.masker != nil { - params.masker.Mask(event.Data) - } - return &seqapi.GetEventResponse{Event: event}, nil + if cached, ok := params.caches.Events.Get(ctx, req.Id); ok { + event := &seqapi.Event{} + if err = proto.Unmarshal([]byte(cached), event); err == nil { + if params.masker != nil { + params.masker.Mask(event.Data) } - logger.Error("failed to unmarshal cached event proto", zap.String("id", req.Id), zap.Error(err)) + return &seqapi.GetEventResponse{Event: event}, nil } + logger.Error("failed to unmarshal cached event proto", zap.String("id", req.Id), zap.Error(err)) } resp, err := params.client.GetEvent(ctx, req) @@ -62,9 +57,9 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap params.masker.Mask(resp.Event.Data) } - if params.eventsCache != nil { + if params.caches.Events != nil { if data, err := proto.Marshal(resp.Event); err == nil { - _ = params.eventsCache.SetWithTTL(ctx, cacheKey, string(data), params.options.Caches.Events.TTL) + params.caches.Events.Set(ctx, req.Id, string(data)) } else { logger.Error("failed to marshal event proto for caching", zap.String("id", req.Id), zap.Error(err)) } diff --git a/internal/api/seqapi/v1/grpc/fetch_async_search_result.go b/internal/api/seqapi/v1/grpc/fetch_async_search_result.go index f96eaa8..a447757 100644 --- a/internal/api/seqapi/v1/grpc/fetch_async_search_result.go +++ b/internal/api/seqapi/v1/grpc/fetch_async_search_result.go @@ -15,15 +15,17 @@ import ( ) func (a *API) FetchAsyncSearchResult(ctx context.Context, req *seqapi.FetchAsyncSearchResultRequest) (*seqapi.FetchAsyncSearchResultResponse, error) { - asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] - if !ok { + if a.asyncSearchesByEnv == nil { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } ctx, span := tracing.StartSpan(ctx, "seqapi_v1_fetch_async_search_result") defer span.End() - spanAttributes := []attribute.KeyValue{ + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + + attributes := []attribute.KeyValue{ { Key: "search_id", Value: attribute.StringValue(req.SearchId), @@ -41,13 +43,16 @@ func (a *API) FetchAsyncSearchResult(ctx context.Context, req *seqapi.FetchAsync Value: attribute.StringValue(string(req.Order)), }, } - span.SetAttributes(spanAttributes...) + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) if _, err := uuid.FromString(req.SearchId); err != nil { return nil, status.Error(codes.InvalidArgument, "invalid search_id") } - resp, err := asyncSearches.FetchAsyncSearchResult(ctx, req) + resp, err := svc.FetchAsyncSearchResult(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/fields.go b/internal/api/seqapi/v1/grpc/fields.go index 281756b..7bc535c 100644 --- a/internal/api/seqapi/v1/grpc/fields.go +++ b/internal/api/seqapi/v1/grpc/fields.go @@ -2,7 +2,6 @@ package grpc import ( "context" - "time" "go.uber.org/zap" "google.golang.org/grpc/codes" @@ -10,47 +9,11 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) -type fieldsCache struct { - cache cache.Cache -} - -func newFieldsCache(c cache.Cache) *fieldsCache { - return &fieldsCache{cache: c} -} - -func (c *fieldsCache) get(ctx context.Context, key string) ([]*seqapi.Field, bool) { - data, err := c.cache.Get(ctx, key) - if err != nil { - return nil, false - } - - var resp seqapi.GetFieldsResponse - if err := proto.Unmarshal([]byte(data), &resp); err != nil { - logger.Error("can't unmarshal cached fields", zap.Error(err)) - return nil, false - } - - return resp.GetFields(), true -} - -func (c *fieldsCache) set(ctx context.Context, key string, fields []*seqapi.Field, ttl time.Duration) { - data, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: fields}) - if err != nil { - logger.Error("can't marshal fields for cache", zap.Error(err)) - return - } - - if err := c.cache.SetWithTTL(ctx, key, string(data), ttl); err != nil { - logger.Error("can't set fields to cache", zap.Error(err)) - } -} - func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seqapi.GetFieldsResponse, error) { ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_fields") defer span.End() @@ -61,23 +24,16 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq return nil, status.Error(codes.InvalidArgument, err.Error()) } - if params.fieldsCache == nil { - resp, err := params.client.GetFields(ctx, req) - if err != nil { - return nil, err + if cached, ok := params.caches.Fields.Get(ctx, ""); ok { + var resp seqapi.GetFieldsResponse + if err = proto.Unmarshal([]byte(cached), &resp); err == nil { + return &seqapi.GetFieldsResponse{ + Fields: resp.GetFields(), + SystemFields: params.systemFields, + PinnedFields: params.pinnedFields, + }, nil } - - resp.SystemFields = params.systemFields - resp.PinnedFields = params.pinnedFields - return resp, nil - } - - if cachedFields, ok := params.fieldsCache.get(ctx, params.fieldsCacheKey); ok { - return &seqapi.GetFieldsResponse{ - Fields: cachedFields, - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, - }, nil + logger.Error("failed to unmarshal cached fields", zap.Error(err)) } resp, err := params.client.GetFields(ctx, req) @@ -85,7 +41,14 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq return nil, err } - params.fieldsCache.set(ctx, params.fieldsCacheKey, resp.GetFields(), params.options.Caches.Fields.TTL) + if params.caches.Fields != nil { + if data, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: resp.GetFields()}); err == nil { + params.caches.Fields.Set(ctx, "", string(data)) + } else { + logger.Error("failed to marshal fields proto for caching", zap.Error(err)) + } + } + resp.SystemFields = params.systemFields resp.PinnedFields = params.pinnedFields return resp, nil diff --git a/internal/api/seqapi/v1/grpc/get_async_searches_list.go b/internal/api/seqapi/v1/grpc/get_async_searches_list.go index 47a1fc2..94175ed 100644 --- a/internal/api/seqapi/v1/grpc/get_async_searches_list.go +++ b/internal/api/seqapi/v1/grpc/get_async_searches_list.go @@ -14,15 +14,17 @@ import ( ) func (a *API) GetAsyncSearchesList(ctx context.Context, req *seqapi.GetAsyncSearchesListRequest) (*seqapi.GetAsyncSearchesListResponse, error) { - asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] - if !ok { + if a.asyncSearchesByEnv == nil { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_async_searches_list") defer span.End() - spanAttributes := []attribute.KeyValue{ + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + + attributes := []attribute.KeyValue{ { Key: "offset", Value: attribute.IntValue(int(req.Offset)), @@ -33,25 +35,27 @@ func (a *API) GetAsyncSearchesList(ctx context.Context, req *seqapi.GetAsyncSear }, } if req.Status != nil { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "status", Value: attribute.StringValue(string(*req.Status)), }) } if req.OwnerName != nil { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "owner", Value: attribute.StringValue(*req.OwnerName), }) } - - span.SetAttributes(spanAttributes...) + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) if err := checkLimitOffset(req.Limit, req.Offset); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - resp, err := asyncSearches.GetAsyncSearchesList(ctx, req) + resp, err := svc.GetAsyncSearchesList(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/histogram.go b/internal/api/seqapi/v1/grpc/histogram.go index 63d6805..261fbd4 100644 --- a/internal/api/seqapi/v1/grpc/histogram.go +++ b/internal/api/seqapi/v1/grpc/histogram.go @@ -40,11 +40,9 @@ func (a *API) GetHistogram(ctx context.Context, req *seqapi.GetHistogramRequest) Value: attribute.IntValue(int(req.GetDownsample())), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) params, err := a.GetParams(env) diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan.go b/internal/api/seqapi/v1/grpc/logs_lifespan.go index 9648325..d8a17d2 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan.go @@ -15,7 +15,6 @@ import ( "github.com/ozontech/seq-ui/internal/api/grpcutil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/lifespan" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" @@ -31,19 +30,14 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ return nil, status.Error(codes.InvalidArgument, err.Error()) } - if params.logsLifespanCache != nil { - if countStr, err := params.logsLifespanCache.Get(ctx, params.logsLifespanCacheKey); err == nil { - count, err := strconv.Atoi(countStr) - if err == nil { - res := time.Duration(count) * lifespan.MeasureUnit - logger.Debug("got logs lifespan from cache") - return &seqapi.GetLogsLifespanResponse{Lifespan: durationpb.New(res)}, nil - } - - logger.Error("can't parse logs lifespan", zap.Error(err)) - } else if !errors.Is(err, cache.ErrNotFound) { - logger.Error("can't get logs lifespan from cache", zap.Error(err)) + if countStr, ok := params.caches.LogsLifespan.Get(ctx, ""); ok { + count, err := strconv.Atoi(countStr) + if err == nil { + res := time.Duration(count) * lifespan.MeasureUnit + logger.Debug("got logs lifespan from cache") + return &seqapi.GetLogsLifespanResponse{Lifespan: durationpb.New(res)}, nil } + logger.Error("failed to parse logs lifespan", zap.Error(err)) } clientStatus, err := params.client.Status(ctx, &seqapi.StatusRequest{}) @@ -58,12 +52,7 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ count := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) res := time.Duration(count) * lifespan.MeasureUnit - if params.logsLifespanCache != nil { - err = params.logsLifespanCache.SetWithTTL(ctx, params.logsLifespanCacheKey, strconv.Itoa(count), params.options.Caches.LogsLifespan.TTL) - if err != nil { - logger.Error("can't set logs lifespan to cache", zap.Error(err)) - } - } + params.caches.LogsLifespan.Set(ctx, "", strconv.Itoa(count)) logger.Debug("got logs lifespan from seq-proxy") return &seqapi.GetLogsLifespanResponse{Lifespan: durationpb.New(res)}, nil diff --git a/internal/api/seqapi/v1/grpc/search.go b/internal/api/seqapi/v1/grpc/search.go index 943c979..6ca4649 100644 --- a/internal/api/seqapi/v1/grpc/search.go +++ b/internal/api/seqapi/v1/grpc/search.go @@ -21,7 +21,7 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se env := types.GetEnvKey(ctx) - spanAttributes := []attribute.KeyValue{ + attributes := []attribute.KeyValue{ { Key: "query", Value: attribute.StringValue(req.GetQuery()), @@ -59,33 +59,30 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se Value: attribute.IntValue(int(req.GetDownsample())), }, } - if env != "" { - spanAttributes = append(spanAttributes, attribute.String("env", env)) + attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(spanAttributes...) - params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } if req.Histogram != nil && req.Histogram.Interval != "" { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "histogram_interval", Value: attribute.StringValue(req.Histogram.Interval), }) } if len(req.Aggregations) > 0 { aggregations, _ := json.Marshal(req.Aggregations) - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "aggregations", Value: attribute.StringValue(string(aggregations)), }) } - span.SetAttributes(spanAttributes...) + span.SetAttributes(attributes...) if err := api_error.CheckSearchLimit(req.Limit, params.options.Limits.MaxSearchLimit); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/internal/api/seqapi/v1/grpc/start_async_search.go b/internal/api/seqapi/v1/grpc/start_async_search.go index d55c316..ae1d419 100644 --- a/internal/api/seqapi/v1/grpc/start_async_search.go +++ b/internal/api/seqapi/v1/grpc/start_async_search.go @@ -15,15 +15,17 @@ import ( ) func (a *API) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSearchRequest) (*seqapi.StartAsyncSearchResponse, error) { - asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] - if !ok { + if a.asyncSearchesByEnv == nil { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } ctx, span := tracing.StartSpan(ctx, "seqapi_v1_start_async_search") defer span.End() - spanAttributes := []attribute.KeyValue{ + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + + attributes := []attribute.KeyValue{ { Key: "query", Value: attribute.StringValue(req.GetQuery()), @@ -50,22 +52,24 @@ func (a *API) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSearch }, } if req.Hist != nil && req.Hist.Interval != "" { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "histogram_interval", Value: attribute.StringValue(req.Hist.Interval), }) } if len(req.Aggs) > 0 { aggregations, _ := json.Marshal(req.Aggs) - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "aggregations", Value: attribute.StringValue(string(aggregations)), }) } + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + span.SetAttributes(attributes...) - span.SetAttributes(spanAttributes...) - - resp, err := asyncSearches.StartAsyncSearch(ctx, req) + resp, err := svc.StartAsyncSearch(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index 6987adc..a7caef0 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -4,6 +4,7 @@ import ( "errors" "time" + handlercache "github.com/ozontech/seq-ui/internal/api/seqapi/v1/handler_cache" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" @@ -19,28 +20,35 @@ var ( func setupTestAPI(data test.APITestData) *API { seqDBClients := make(map[string]seqdb.Client, max(1, len(data.Cfg.Envs))) - asyncSearhes := make(map[string]asyncsearches.Service, max(1, len(data.Cfg.Envs))) + asyncSearhesByEnv := make(map[string]asyncsearches.Service, max(1, len(data.Cfg.Envs))) if len(data.Cfg.Envs) == 0 { seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB - asyncSearhes[""] = data.Mocks.AsyncSearchesSvc + asyncSearhesByEnv[""] = data.Mocks.AsyncSearchesSvc } for envName, envConfig := range data.Cfg.Envs { seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB - asyncSearhes[envName] = data.Mocks.AsyncSearchesSvc + asyncSearhesByEnv[envName] = data.Mocks.AsyncSearchesSvc } if data.Mocks.AsyncSearchesSvc == nil { - asyncSearhes = nil + asyncSearhesByEnv = nil } - api := New(&data.Cfg, seqDBClients, asyncSearhes) + api := New(&data.Cfg, seqDBClients, asyncSearhesByEnv) if data.Mocks.Cache != nil { + caches := data.Cfg.Options.Caches for env, params := range api.paramsByEnv { - params.eventsCache = data.Mocks.Cache - params.logsLifespanCache = data.Mocks.Cache - params.fieldsCache = newFieldsCache(data.Mocks.Cache) + if caches.Events != nil { + params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: caches.Events.TTL} + } + if caches.Fields != nil { + params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: caches.Fields.TTL} + } + if caches.LogsLifespan != nil { + params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: caches.LogsLifespan.TTL} + } api.paramsByEnv[env] = params } } diff --git a/internal/api/seqapi/v1/handler_cache/handler_cache.go b/internal/api/seqapi/v1/handler_cache/handler_cache.go new file mode 100644 index 0000000..6c0ec1a --- /dev/null +++ b/internal/api/seqapi/v1/handler_cache/handler_cache.go @@ -0,0 +1,77 @@ +package handlercache + +import ( + "context" + "errors" + "time" + + "go.uber.org/zap" + + "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/pkg/cache" + "github.com/ozontech/seq-ui/logger" +) + +type Cache struct { + Cache cache.Cache + KeyPrefix string + TTL time.Duration +} + +type Caches struct { + Events *Cache + Fields *Cache + LogsLifespan *Cache +} + +func New(env string, cfg config.SeqAPICaches) Caches { + keyPrefix := "seqapi_" + if env != "" { + keyPrefix += env + "_" + } + + return Caches{ + Events: newCache(cfg.Events, keyPrefix+"event_"), + Fields: newCache(cfg.Fields, keyPrefix+"fields"), + LogsLifespan: newCache(cfg.LogsLifespan, keyPrefix+"logs_lifespan"), + } +} + +func newCache(cfg *config.HandlerCache, keyPrefix string) *Cache { + if cfg == nil { + return nil + } + + c, ok := cache.ByID(cfg.ID) + if !ok { + return nil + } + + return &Cache{Cache: c, KeyPrefix: keyPrefix, TTL: cfg.TTL} +} + +func (c *Cache) Get(ctx context.Context, key string) (string, bool) { + if c == nil { + return "", false + } + + value, err := c.Cache.Get(ctx, c.KeyPrefix+key) + if err != nil { + if !errors.Is(err, cache.ErrNotFound) { + logger.Error("failed to get value from cache", zap.String("key", c.KeyPrefix+key), zap.Error(err)) + } + return "", false + } + + return value, true +} + +func (c *Cache) Set(ctx context.Context, key, value string) { + if c == nil { + return + } + + if err := c.Cache.SetWithTTL(ctx, c.KeyPrefix+key, value, c.TTL); err != nil { + logger.Error("failed to set value to cache", zap.String("key", c.KeyPrefix+key), zap.Error(err)) + } +} diff --git a/internal/api/seqapi/v1/http/aggregation.go b/internal/api/seqapi/v1/http/aggregation.go index 3a5dab8..3195a82 100644 --- a/internal/api/seqapi/v1/http/aggregation.go +++ b/internal/api/seqapi/v1/http/aggregation.go @@ -69,11 +69,9 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { Value: attribute.IntValue(int(httpReq.Downsample)), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) params, err := a.GetEnvParams(env) diff --git a/internal/api/seqapi/v1/http/aggregation_ts.go b/internal/api/seqapi/v1/http/aggregation_ts.go index 130d9e7..9fa4206 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts.go +++ b/internal/api/seqapi/v1/http/aggregation_ts.go @@ -63,11 +63,9 @@ func (a *API) serveGetAggregationTs(w http.ResponseWriter, r *http.Request) { Value: attribute.StringValue(string(aggsRaw)), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) params, err := a.GetEnvParams(env) diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index 0fc12f3..84a30a7 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -2,6 +2,8 @@ package http import ( "fmt" + "maps" + "slices" "sort" "strconv" "time" @@ -10,10 +12,10 @@ import ( "github.com/gofrs/uuid" "go.uber.org/zap" + handlercache "github.com/ozontech/seq-ui/internal/api/seqapi/v1/handler_cache" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/tokenlimiter" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/mask" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" @@ -22,48 +24,40 @@ import ( ) type apiParams struct { - client seqdb.Client - options *config.SeqAPIOptions - masker *mask.Masker - pinnedFields fields - systemFields fields - exportLimiter *tokenlimiter.Limiter - eventsCache cache.Cache - logsLifespanCache cache.Cache - fieldsCache *fieldsCache - eventsCacheKeyPrefix string - fieldsCacheKey string - logsLifespanCacheKey string + client seqdb.Client + options *config.SeqAPIOptions + masker *mask.Masker + pinnedFields fields + systemFields fields + exportLimiter *tokenlimiter.Limiter + caches handlercache.Caches } type API struct { - config *config.SeqAPI - paramsByEnv map[string]apiParams - nowFn func() time.Time - asyncSearches map[string]asyncsearches.Service - envsResponse getEnvsResponse + config *config.SeqAPI + paramsByEnv map[string]apiParams + nowFn func() time.Time + asyncSearchesByEnv map[string]asyncsearches.Service + envsResponse getEnvsResponse } -func New(cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, asyncSearches map[string]asyncsearches.Service) *API { +func New(cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, asyncSearchesByEnv map[string]asyncsearches.Service) *API { paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { for envName, envConfig := range cfg.Envs { paramsByEnv[envName] = newAPIParams(envName, seqDBСlients[envConfig.SeqDBID], envConfig.Options) } } else { - var client seqdb.Client - for _, c := range seqDBСlients { - client = c - } + client := slices.Collect(maps.Values(seqDBСlients))[0] paramsByEnv[""] = newAPIParams("", client, &cfg.Options) } return &API{ - config: cfg, - paramsByEnv: paramsByEnv, - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + config: cfg, + paramsByEnv: paramsByEnv, + nowFn: time.Now, + asyncSearchesByEnv: asyncSearchesByEnv, + envsResponse: parseEnvs(cfg), } } @@ -78,39 +72,14 @@ func newAPIParams(env string, client seqdb.Client, options *config.SeqAPIOptions client.WithMasking(masker) } - var eventsCache, logsLifespanCache cache.Cache - if options.Caches.Events != nil { - eventsCache, _ = cache.ByID(options.Caches.Events.ID) - } - if options.Caches.LogsLifespan != nil { - logsLifespanCache, _ = cache.ByID(options.Caches.LogsLifespan.ID) - } - - var fc *fieldsCache - if options.Caches.Fields != nil { - if c, ok := cache.ByID(options.Caches.Fields.ID); ok { - fc = newFieldsCache(c) - } - } - - cacheKeyPrefix := "seqapi_" - if env != "" { - cacheKeyPrefix += env + "_" - } - return apiParams{ - client: client, - options: options, - masker: masker, - pinnedFields: parseFields(options.PinnedFields), - systemFields: parseFields(options.SystemFields), - exportLimiter: tokenlimiter.New(options.Limits.MaxParallelExportRequests), - eventsCache: eventsCache, - logsLifespanCache: logsLifespanCache, - fieldsCache: fc, - eventsCacheKeyPrefix: cacheKeyPrefix + "event_", - fieldsCacheKey: cacheKeyPrefix + "fields", - logsLifespanCacheKey: cacheKeyPrefix + "logs_lifespan", + client: client, + options: options, + masker: masker, + pinnedFields: parseFields(options.PinnedFields), + systemFields: parseFields(options.SystemFields), + exportLimiter: tokenlimiter.New(options.Limits.MaxParallelExportRequests), + caches: handlercache.New(env, options.Caches), } } diff --git a/internal/api/seqapi/v1/http/cancel_async_search.go b/internal/api/seqapi/v1/http/cancel_async_search.go index f037a85..2e984c3 100644 --- a/internal/api/seqapi/v1/http/cancel_async_search.go +++ b/internal/api/seqapi/v1/http/cancel_async_search.go @@ -24,21 +24,20 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveCancelAsyncSearch(w http.ResponseWriter, r *http.Request) { - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_cancel_async_search") - defer span.End() - wr := httputil.NewWriter(w) - env := types.GetEnvKey(ctx) - asyncSearches, ok := a.asyncSearches[env] - if !ok { + if a.asyncSearchesByEnv == nil { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_cancel_async_search") + defer span.End() + + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + searchID := chi.URLParam(r, "id") - // я бы в service вынес проверочку и попробовал перейти на стандартную либу uuid(она вроде с этой версии появилось) - // https://go.dev/doc/go1.27#uuid такая вот фигня if err := checkUUID(searchID); err != nil { wr.Error(err, http.StatusBadRequest) return @@ -50,14 +49,12 @@ func (a *API) serveCancelAsyncSearch(w http.ResponseWriter, r *http.Request) { Value: attribute.StringValue(searchID), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) - _, err := asyncSearches.CancelAsyncSearch(ctx, &seqapi.CancelAsyncSearchRequest{SearchId: searchID}) + _, err := svc.CancelAsyncSearch(ctx, &seqapi.CancelAsyncSearchRequest{SearchId: searchID}) if err != nil { status := http.StatusInternalServerError if errors.Is(err, types.ErrPermissionDenied) { diff --git a/internal/api/seqapi/v1/http/delete_async_search.go b/internal/api/seqapi/v1/http/delete_async_search.go index 406d78a..f8c6727 100644 --- a/internal/api/seqapi/v1/http/delete_async_search.go +++ b/internal/api/seqapi/v1/http/delete_async_search.go @@ -24,20 +24,20 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveDeleteAsyncSearch(w http.ResponseWriter, r *http.Request) { - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_delete_async_search") - defer span.End() - wr := httputil.NewWriter(w) - env := types.GetEnvKey(ctx) - asyncSearches, ok := a.asyncSearches[env] - if !ok { + if a.asyncSearchesByEnv == nil { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_delete_async_search") + defer span.End() + + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + searchID := chi.URLParam(r, "id") - // в cancel_async описал, чтобы я с этой проверкой сделал if err := checkUUID(searchID); err != nil { wr.Error(err, http.StatusBadRequest) return @@ -49,14 +49,12 @@ func (a *API) serveDeleteAsyncSearch(w http.ResponseWriter, r *http.Request) { Value: attribute.StringValue(searchID), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) - _, err := asyncSearches.DeleteAsyncSearch(ctx, &seqapi.DeleteAsyncSearchRequest{SearchId: searchID}) + _, err := svc.DeleteAsyncSearch(ctx, &seqapi.DeleteAsyncSearchRequest{SearchId: searchID}) if err != nil { status := http.StatusInternalServerError if errors.Is(err, types.ErrPermissionDenied) { diff --git a/internal/api/seqapi/v1/http/events.go b/internal/api/seqapi/v1/http/events.go index c9c725b..a1dcc0b 100644 --- a/internal/api/seqapi/v1/http/events.go +++ b/internal/api/seqapi/v1/http/events.go @@ -40,11 +40,9 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { Value: attribute.StringValue(id), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) params, err := a.GetEnvParams(env) @@ -53,19 +51,16 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { return } - cacheKey := params.eventsCacheKeyPrefix + id // как лучше с ним поступить не придумал - if params.eventsCache != nil { - if cached, err := params.eventsCache.Get(ctx, cacheKey); err == nil { - e := &seqapi.Event{} - if err = proto.Unmarshal([]byte(cached), e); err == nil { - if params.masker != nil { - params.masker.Mask(e.Data) - } - wr.WriteJson(getEventResponseFromProto(&seqapi.GetEventResponse{Event: e})) - return + if cached, ok := params.caches.Events.Get(ctx, id); ok { + e := &seqapi.Event{} + if err = proto.Unmarshal([]byte(cached), e); err == nil { + if params.masker != nil { + params.masker.Mask(e.Data) } - logger.Error("failed to unmarshal cached event proto", zap.String("id", id), zap.Error(err)) + wr.WriteJson(getEventResponseFromProto(&seqapi.GetEventResponse{Event: e})) + return } + logger.Error("failed to unmarshal cached event proto", zap.String("id", id), zap.Error(err)) } resp, err := params.client.GetEvent(ctx, &seqapi.GetEventRequest{Id: id}) @@ -78,9 +73,9 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { params.masker.Mask(resp.Event.Data) } - if params.eventsCache != nil { + if params.caches.Events != nil { if data, err := proto.Marshal(resp.Event); err == nil { - _ = params.eventsCache.SetWithTTL(ctx, cacheKey, string(data), params.options.Caches.Events.TTL) + params.caches.Events.Set(ctx, id, string(data)) } else { logger.Error("failed to marshal event proto for caching", zap.String("id", id), zap.Error(err)) } diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index b1a3d3b..7682771 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -113,7 +113,6 @@ func TestServeGetEvent(t *testing.T) { }, }, } - // убрать фатал при пинге(пинговать, но не фаталиться) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() diff --git a/internal/api/seqapi/v1/http/export.go b/internal/api/seqapi/v1/http/export.go index 9e10401..7cee32a 100644 --- a/internal/api/seqapi/v1/http/export.go +++ b/internal/api/seqapi/v1/http/export.go @@ -83,11 +83,9 @@ func (a *API) serveExport(w http.ResponseWriter, r *http.Request) { Value: attribute.IntValue(int(httpReq.Downsample)), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) params, err := a.GetEnvParams(env) diff --git a/internal/api/seqapi/v1/http/export_async_search.go b/internal/api/seqapi/v1/http/export_async_search.go index f71a506..26e0a15 100644 --- a/internal/api/seqapi/v1/http/export_async_search.go +++ b/internal/api/seqapi/v1/http/export_async_search.go @@ -26,18 +26,19 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_export_async_search_http") - defer span.End() - wr := httputil.NewWriter(w) - env := types.GetEnvKey(ctx) - asyncSearches, ok := a.asyncSearches[env] - if !ok { + if a.asyncSearchesByEnv == nil { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_export_async_search_http") + defer span.End() + + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + userStr := "_" if userName, err := types.GetUserKey(ctx); err == nil { userStr = userName @@ -75,11 +76,9 @@ func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { Value: attribute.StringSliceValue(httpReq.Fields), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) if err := checkUUID(httpReq.SearchID); err != nil { @@ -123,7 +122,7 @@ func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { return } - if err := asyncSearches.ExportAsyncSearch(ctx, httpReq.toProto(), cw); err != nil { + if err := svc.ExportAsyncSearch(ctx, httpReq.toProto(), cw); err != nil { wr.Error(err, http.StatusInternalServerError) return } diff --git a/internal/api/seqapi/v1/http/fetch_async_search_result.go b/internal/api/seqapi/v1/http/fetch_async_search_result.go index 14fa485..945aa78 100644 --- a/internal/api/seqapi/v1/http/fetch_async_search_result.go +++ b/internal/api/seqapi/v1/http/fetch_async_search_result.go @@ -26,25 +26,26 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveFetchAsyncSearchResult(w http.ResponseWriter, r *http.Request) { - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_fetch_async_search_result") - defer span.End() - wr := httputil.NewWriter(w) - env := types.GetEnvKey(ctx) - asyncSearches, ok := a.asyncSearches[env] - if !ok { + if a.asyncSearchesByEnv == nil { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_fetch_async_search_result") + defer span.End() + + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + var httpReq fetchAsyncSearchResultRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) return } - spanAttributes := []attribute.KeyValue{ + attributes := []attribute.KeyValue{ { Key: "search_id", Value: attribute.StringValue(httpReq.SearchID), @@ -62,12 +63,10 @@ func (a *API) serveFetchAsyncSearchResult(w http.ResponseWriter, r *http.Request Value: attribute.StringValue(string(httpReq.Order)), }, } - if env != "" { - spanAttributes = append(spanAttributes, attribute.String("env", env)) + attributes = append(attributes, attribute.String("env", env)) } - - span.SetAttributes(spanAttributes...) + span.SetAttributes(attributes...) if err := checkUUID(httpReq.SearchID); err != nil { wr.Error(err, http.StatusBadRequest) @@ -79,7 +78,7 @@ func (a *API) serveFetchAsyncSearchResult(w http.ResponseWriter, r *http.Request return } - resp, err := asyncSearches.FetchAsyncSearchResult(ctx, httpReq.toProto()) + resp, err := svc.FetchAsyncSearchResult(ctx, httpReq.toProto()) if err != nil { wr.Error(err, http.StatusInternalServerError) return diff --git a/internal/api/seqapi/v1/http/fields.go b/internal/api/seqapi/v1/http/fields.go index f89a9ab..d5a5fb2 100644 --- a/internal/api/seqapi/v1/http/fields.go +++ b/internal/api/seqapi/v1/http/fields.go @@ -1,57 +1,19 @@ package http import ( - "context" "net/http" - "time" "go.uber.org/zap" "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) -type fieldsCache struct { - cache cache.Cache -} - -func newFieldsCache(c cache.Cache) *fieldsCache { - return &fieldsCache{cache: c} -} - -func (c *fieldsCache) get(ctx context.Context, key string) ([]*seqapi.Field, bool) { - data, err := c.cache.Get(ctx, key) - if err != nil { - return nil, false - } - - var resp seqapi.GetFieldsResponse - if err := proto.Unmarshal([]byte(data), &resp); err != nil { - logger.Error("can't unmarshal cached fields", zap.Error(err)) - return nil, false - } - - return resp.GetFields(), true -} - -func (c *fieldsCache) set(ctx context.Context, key string, fields []*seqapi.Field, ttl time.Duration) { - data, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: fields}) - if err != nil { - logger.Error("can't marshal fields for cache", zap.Error(err)) - return - } - - if err := c.cache.SetWithTTL(ctx, key, string(data), ttl); err != nil { - logger.Error("can't set fields to cache", zap.Error(err)) - } -} - // serveGetFields go doc. // // @Router /seqapi/v1/fields [get] @@ -73,30 +35,17 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - if params.fieldsCache == nil { - resp, err := params.client.GetFields(ctx, &seqapi.GetFieldsRequest{}) - if err != nil { - wr.Error(err, http.StatusInternalServerError) + if cached, ok := params.caches.Fields.Get(ctx, ""); ok { + var resp seqapi.GetFieldsResponse + if err = proto.Unmarshal([]byte(cached), &resp); err == nil { + wr.WriteJson(getFieldsResponse{ + Fields: fieldsFromProto(resp.GetFields()), + SystemFields: params.systemFields, + PinnedFields: params.pinnedFields, + }) return } - - res := getFieldsResponse{ - Fields: fieldsFromProto(resp.GetFields()), - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, - } - wr.WriteJson(res) - return - } - - if cachedFields, ok := params.fieldsCache.get(ctx, params.fieldsCacheKey); ok { - res := getFieldsResponse{ - Fields: fieldsFromProto(cachedFields), - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, - } - wr.WriteJson(res) - return + logger.Error("failed to unmarshal cached fields", zap.Error(err)) } resp, err := params.client.GetFields(ctx, &seqapi.GetFieldsRequest{}) @@ -105,14 +54,19 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - params.fieldsCache.set(ctx, params.fieldsCacheKey, resp.GetFields(), params.options.Caches.Fields.TTL) - res := getFieldsResponse{ + if params.caches.Fields != nil { + if data, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: resp.GetFields()}); err == nil { + params.caches.Fields.Set(ctx, "", string(data)) + } else { + logger.Error("failed to marshal fields proto for caching", zap.Error(err)) + } + } + + wr.WriteJson(getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), SystemFields: params.systemFields, PinnedFields: params.pinnedFields, - } - - wr.WriteJson(res) + }) } // serveGetPinnedFields go doc. @@ -146,18 +100,18 @@ type field struct { Type string `json:"type" default:"unknown" enums:"unknown,keyword,text"` } // @name seqapi.v1.Field -func fieldFromProto(proto *seqapi.Field) field { +func fieldFromProto(f *seqapi.Field) field { return field{ - Name: proto.GetName(), - Type: seqdb.FieldTypeFromProto(proto.GetType()), + Name: f.GetName(), + Type: seqdb.FieldTypeFromProto(f.GetType()), } } type fields []field -func fieldsFromProto(proto []*seqapi.Field) fields { - res := make(fields, len(proto)) - for i, f := range proto { +func fieldsFromProto(fs []*seqapi.Field) fields { + res := make(fields, len(fs)) + for i, f := range fs { res[i] = fieldFromProto(f) } return res diff --git a/internal/api/seqapi/v1/http/get_async_searches_list.go b/internal/api/seqapi/v1/http/get_async_searches_list.go index efdc025..8082f16 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list.go @@ -26,25 +26,26 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) { - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_async_searches_list") - defer span.End() - wr := httputil.NewWriter(w) - env := types.GetEnvKey(ctx) - asyncSearches, ok := a.asyncSearches[env] - if !ok { + if a.asyncSearchesByEnv == nil { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_async_searches_list") + defer span.End() + + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + var httpReq getAsyncSearchesListRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) return } - spanAttributes := []attribute.KeyValue{ + attributes := []attribute.KeyValue{ { Key: "offset", Value: attribute.IntValue(int(httpReq.Offset)), @@ -55,22 +56,21 @@ func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) }, } if httpReq.Status != nil { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "status", Value: attribute.StringValue(string(*httpReq.Status)), }) } if httpReq.Owner != nil { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "owner", Value: attribute.StringValue(*httpReq.Owner), }) } if env != "" { - spanAttributes = append(spanAttributes, attribute.String("env", env)) + attributes = append(attributes, attribute.String("env", env)) } - - span.SetAttributes(spanAttributes...) + span.SetAttributes(attributes...) if err := checkLimitOffset(httpReq.Limit, httpReq.Offset); err != nil { wr.Error(err, http.StatusBadRequest) @@ -83,7 +83,7 @@ func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) return } - resp, err := asyncSearches.GetAsyncSearchesList(ctx, parsedReq) + resp, err := svc.GetAsyncSearchesList(ctx, parsedReq) if err != nil { wr.Error(err, http.StatusInternalServerError) return diff --git a/internal/api/seqapi/v1/http/histogram.go b/internal/api/seqapi/v1/http/histogram.go index 72717e7..8db4d20 100644 --- a/internal/api/seqapi/v1/http/histogram.go +++ b/internal/api/seqapi/v1/http/histogram.go @@ -62,11 +62,9 @@ func (a *API) serveGetHistogram(w http.ResponseWriter, r *http.Request) { Value: attribute.IntValue(int(httpReq.Downsample)), }, } - if env != "" { attributes = append(attributes, attribute.String("env", env)) } - span.SetAttributes(attributes...) params, err := a.GetEnvParams(env) diff --git a/internal/api/seqapi/v1/http/logs_lifespan.go b/internal/api/seqapi/v1/http/logs_lifespan.go index 0f55aa8..ebf38e5 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan.go +++ b/internal/api/seqapi/v1/http/logs_lifespan.go @@ -11,7 +11,6 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/lifespan" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" @@ -38,19 +37,14 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { return } - if params.logsLifespanCache != nil { - if resStr, err := params.logsLifespanCache.Get(ctx, params.logsLifespanCacheKey); err == nil { - res, err := strconv.Atoi(resStr) - if err == nil { - logger.Debug("got logs lifespan from cache") - wr.WriteJson(getLogsLifespanResponse{Lifespan: res}) - return - } - - logger.Error("can't parse logs lifespan", zap.Error(err)) - } else if !errors.Is(err, cache.ErrNotFound) { - logger.Error("can't get logs lifespan from cache", zap.Error(err)) + if resStr, ok := params.caches.LogsLifespan.Get(ctx, ""); ok { + res, err := strconv.Atoi(resStr) + if err == nil { + logger.Debug("got logs lifespan from cache") + wr.WriteJson(getLogsLifespanResponse{Lifespan: res}) + return } + logger.Error("failed to parse logs lifespan", zap.Error(err)) } clientStatus, err := params.client.Status(ctx, &seqapi.StatusRequest{}) @@ -66,15 +60,7 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { res := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) - if params.logsLifespanCache != nil { - if err = params.logsLifespanCache.SetWithTTL( - ctx, - params.logsLifespanCacheKey, - strconv.Itoa(res), - params.options.Caches.LogsLifespan.TTL); err != nil { - logger.Error("can't set logs lifespan to cache", zap.Error(err)) - } - } + params.caches.LogsLifespan.Set(ctx, "", strconv.Itoa(res)) logger.Debug("got logs lifespan from seq-proxy") wr.WriteJson(getLogsLifespanResponse{Lifespan: res}) diff --git a/internal/api/seqapi/v1/http/search.go b/internal/api/seqapi/v1/http/search.go index 5e330fc..3285fd3 100644 --- a/internal/api/seqapi/v1/http/search.go +++ b/internal/api/seqapi/v1/http/search.go @@ -45,7 +45,7 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { httpReq.Order = oDESC } - spanAttributes := []attribute.KeyValue{ + attributes := []attribute.KeyValue{ { Key: "query", Value: attribute.StringValue(httpReq.Query), @@ -83,9 +83,8 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { Value: attribute.IntValue(int(httpReq.Downsample)), }, } - if env != "" { - spanAttributes = append(spanAttributes, attribute.String("env", env)) + attributes = append(attributes, attribute.String("env", env)) } params, err := a.GetEnvParams(env) @@ -95,20 +94,19 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { } if httpReq.Histogram.Interval != "" { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "histogram_interval", Value: attribute.StringValue(httpReq.Histogram.Interval), }) } if len(httpReq.Aggregations) > 0 { aggsRaw, _ := json.Marshal(httpReq.Aggregations) - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "aggregations", Value: attribute.StringValue(string(aggsRaw)), }) } - - span.SetAttributes(spanAttributes...) + span.SetAttributes(attributes...) if err := api_error.CheckSearchLimit(httpReq.Limit, params.options.Limits.MaxSearchLimit); err != nil { wr.Error(err, http.StatusBadRequest) @@ -187,8 +185,8 @@ func (r searchRequest) toProto() *seqapi.SearchRequest { Query: r.Query, From: timestamppb.New(r.From), To: timestamppb.New(r.To), - Limit: int32(r.Limit), - Offset: int32(r.Offset), + Limit: r.Limit, + Offset: r.Offset, OffsetId: r.OffsetID, WithTotal: r.WithTotal, Downsample: r.Downsample, diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index 31543e0..d70ac92 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -28,18 +28,19 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_start_async_search") - defer span.End() - wr := httputil.NewWriter(w) - env := types.GetEnvKey(ctx) - asyncSearches, ok := a.asyncSearches[env] - if !ok { + if a.asyncSearchesByEnv == nil { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_start_async_search") + defer span.End() + + env := types.GetEnvKey(ctx) + svc := a.asyncSearchesByEnv[env] + params, err := a.GetEnvParams(env) if err != nil { httputil.NewWriter(w).Error(err, http.StatusBadRequest) @@ -70,7 +71,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { } } - spanAttributes := []attribute.KeyValue{ + attributes := []attribute.KeyValue{ { Key: "query", Value: attribute.StringValue(httpReq.Query), @@ -97,25 +98,24 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { }, } if httpReq.Histogram != nil && httpReq.Histogram.Interval != "" { - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "histogram_interval", Value: attribute.StringValue(httpReq.Histogram.Interval), }) } if len(httpReq.Aggregations) > 0 { aggsRaw, _ := json.Marshal(httpReq.Aggregations) - spanAttributes = append(spanAttributes, attribute.KeyValue{ + attributes = append(attributes, attribute.KeyValue{ Key: "aggregations", Value: attribute.StringValue(string(aggsRaw)), }) } if env != "" { - spanAttributes = append(spanAttributes, attribute.String("env", env)) + attributes = append(attributes, attribute.String("env", env)) } + span.SetAttributes(attributes...) - span.SetAttributes(spanAttributes...) - - resp, err := asyncSearches.StartAsyncSearch(ctx, httpReq.toProto(parsedRetention)) + resp, err := svc.StartAsyncSearch(ctx, httpReq.toProto(parsedRetention)) if err != nil { wr.Error(err, http.StatusInternalServerError) return diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 43dadde..17d5f5d 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -8,6 +8,7 @@ import ( "github.com/go-chi/chi/v5" + handlercache "github.com/ozontech/seq-ui/internal/api/seqapi/v1/handler_cache" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" @@ -24,28 +25,35 @@ var ( func setupTestAPI(data test.APITestData) *API { seqDBClients := make(map[string]seqdb.Client, max(1, len(data.Cfg.Envs))) - asyncSearhes := make(map[string]asyncsearches.Service, max(1, len(data.Cfg.Envs))) + asyncSearhesByEnv := make(map[string]asyncsearches.Service, max(1, len(data.Cfg.Envs))) if len(data.Cfg.Envs) == 0 { seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB - asyncSearhes[""] = data.Mocks.AsyncSearchesSvc + asyncSearhesByEnv[""] = data.Mocks.AsyncSearchesSvc } for envName, envConfig := range data.Cfg.Envs { seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB - asyncSearhes[envName] = data.Mocks.AsyncSearchesSvc + asyncSearhesByEnv[envName] = data.Mocks.AsyncSearchesSvc } if data.Mocks.AsyncSearchesSvc == nil { - asyncSearhes = nil + asyncSearhesByEnv = nil } - api := New(&data.Cfg, seqDBClients, asyncSearhes) + api := New(&data.Cfg, seqDBClients, asyncSearhesByEnv) if data.Mocks.Cache != nil { + caches := data.Cfg.Options.Caches for env, params := range api.paramsByEnv { - params.eventsCache = data.Mocks.Cache - params.logsLifespanCache = data.Mocks.Cache - params.fieldsCache = newFieldsCache(data.Mocks.Cache) + if caches.Events != nil { + params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: caches.Events.TTL} + } + if caches.Fields != nil { + params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: caches.Fields.TTL} + } + if caches.LogsLifespan != nil { + params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: caches.LogsLifespan.TTL} + } api.paramsByEnv[env] = params } } diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index 82d337d..d5c03bc 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -15,10 +15,10 @@ type SeqAPI struct { httpAPI *http_api.API } -func New(cfg *config.SeqAPI, seqDBClients map[string]seqdb.Client, asyncSearches map[string]asyncsearches.Service) *SeqAPI { +func New(cfg *config.SeqAPI, seqDBClients map[string]seqdb.Client, asyncSearchesByEnv map[string]asyncsearches.Service) *SeqAPI { return &SeqAPI{ - grpcAPI: grpc_api.New(cfg, seqDBClients, asyncSearches), - httpAPI: http_api.New(cfg, seqDBClients, asyncSearches), + grpcAPI: grpc_api.New(cfg, seqDBClients, asyncSearchesByEnv), + httpAPI: http_api.New(cfg, seqDBClients, asyncSearchesByEnv), } } diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index 22150ee..213527e 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -17,27 +17,34 @@ func V1ToV2(src v1.Config) v2.Config { } func (m *v1tov2) migrate() v2.Config { - dst := v2.Config{Version: 2} - dst.Server = m.migrateServer(m.src.Server) + server := m.src.Server + if server == nil { + // in v1 are always enabled + server = &v1.Server{} + } - var v1Cache v1.Cache - if m.src.Server != nil { - v1Cache = m.src.Server.Cache + handlers := m.src.Handlers + if handlers == nil { + // in v1 are always enabled + handlers = &v1.Handlers{} } - dst.Cache = m.migrateCache(v1Cache) - dst.Clients = m.migrateClients(m.src) - dst.DB = m.migrateDB(m.src.Server) - dst.Handlers = m.migrateHandlers(m.src.Handlers, v1Cache) + var v1SessionStore *v1.SessionStore + if m.src.Handlers != nil && m.src.Handlers.MassExport != nil { + v1SessionStore = m.src.Handlers.MassExport.SessionStore + } - return dst + return v2.Config{ + Version: 2, + Server: m.migrateServer(server), + Cache: m.migrateCache(server.Cache, v1SessionStore), + Clients: m.migrateClients(m.src.Clients, server.CH), + DB: m.migrateDB(server.DB), + Handlers: m.migrateHandlers(handlers, server), + } } func (m *v1tov2) migrateServer(src *v1.Server) v2.Server { - if src == nil { - return v2.Server{} - } - dst := v2.Server{ HTTP: &v2.HTTP{ Addr: src.HTTPAddr, @@ -71,6 +78,7 @@ func (m *v1tov2) migrateServer(src *v1.Server) v2.Server { if src.OIDC.CacheSecretKey != "" { cacheID, _ := m.defaultCacheIDs(src.Cache) oidc.CacheID = cacheID + oidc.CacheSecretKey = src.OIDC.CacheSecretKey } dst.Auth = &v2.Auth{Options: v2.AuthOptions{OIDC: oidc}} @@ -168,31 +176,30 @@ func (m *v1tov2) migrateGRPCParams(src *v1.GRPCKeepaliveParams) *v2.GRPCParams { } } -func (m *v1tov2) migrateClients(src v1.Config) v2.Clients { +func (m *v1tov2) migrateClients(src *v1.Clients, ch *v1.CH) v2.Clients { dst := v2.Clients{} - if src.Clients != nil { - if len(src.Clients.SeqDB) > 0 { - dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) - for i := range src.Clients.SeqDB { - dst.SeqDB = append(dst.SeqDB, m.migrateSeqDBClient(&src.Clients.SeqDB[i])) + if src != nil { + if len(src.SeqDB) > 0 { + dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.SeqDB)) + for i := range src.SeqDB { + dst.SeqDB = append(dst.SeqDB, m.migrateSeqDBClient(&src.SeqDB[i])) } } else { dst.SeqDB = []v2.SeqDBClient{{ ID: v2.DefaultSeqDBClientID, - Timeout: src.Clients.SeqDBTimeout, - AvgDocSize: src.Clients.SeqDBAvgDocSize, - Addrs: src.Clients.SeqDBAddrs, - RequestRetries: src.Clients.RequestRetries, - RequestRetriesBackoff: m.migrateBackoff(src.Clients.InitialRetryBackoff, src.Clients.MaxRetryBackoff), - ClientMode: src.Clients.ProxyClientMode, - GRPCParams: m.migrateGRPCParams(src.Clients.GRPCKeepaliveParams), + Timeout: src.SeqDBTimeout, + AvgDocSize: src.SeqDBAvgDocSize, + Addrs: src.SeqDBAddrs, + RequestRetries: src.RequestRetries, + RequestRetriesBackoff: m.migrateBackoff(src.InitialRetryBackoff, src.MaxRetryBackoff), + ClientMode: src.ProxyClientMode, + GRPCParams: m.migrateGRPCParams(src.GRPCKeepaliveParams), }} } } - if src.Server != nil && src.Server.CH != nil { - ch := src.Server.CH + if ch != nil { dst.ClickHouse = []v2.CHClient{{ ID: v2.DefaultCHClientID, Addrs: ch.Addrs, @@ -232,24 +239,24 @@ func (m *v1tov2) migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { } } -func (m *v1tov2) migrateDB(src *v1.Server) *v2.DB { - if src == nil || src.DB == nil { +func (m *v1tov2) migrateDB(src *v1.DB) *v2.DB { + if src == nil { return nil } return &v2.DB{ - Name: src.DB.Name, - Host: src.DB.Host, - Port: src.DB.Port, - User: src.DB.User, - Pass: src.DB.Pass, - RequestTimeout: src.DB.RequestTimeout, - ConnectionPoolCapacity: src.DB.ConnectionPoolCapacity, - UsePreparedStatements: src.DB.UsePreparedStatements, + Name: src.Name, + Host: src.Host, + Port: src.Port, + User: src.User, + Pass: src.Pass, + RequestTimeout: src.RequestTimeout, + ConnectionPoolCapacity: src.ConnectionPoolCapacity, + UsePreparedStatements: src.UsePreparedStatements, } } -func (m *v1tov2) migrateCache(src v1.Cache) v2.Cache { +func (m *v1tov2) migrateCache(src v1.Cache, sessionStore *v1.SessionStore) v2.Cache { // in config v1 inmemory cache is always created cache := v2.Cache{ Inmemory: []v2.Inmemory{{ @@ -267,6 +274,10 @@ func (m *v1tov2) migrateCache(src v1.Cache) v2.Cache { ) } + if sessionStore != nil { + cache.Redis = append(cache.Redis, m.migrateRedis(&sessionStore.Redis, v2.DefaultMassExportRedisID, "")) + } + return cache } @@ -285,24 +296,20 @@ func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { } } -func (m *v1tov2) migrateHandlers(src *v1.Handlers, v1Cache v1.Cache) v2.Handlers { - if src == nil { - src = &v1.Handlers{} - } - +func (m *v1tov2) migrateHandlers(src *v1.Handlers, server *v1.Server) v2.Handlers { dst := v2.Handlers{ - SeqAPI: m.migrateSeqAPI(&src.SeqAPI, v1Cache), + SeqAPI: m.migrateSeqAPI(&src.SeqAPI, server.Cache), AsyncSearch: m.migrateAsyncSearch(&src.AsyncSearch, &src.SeqAPI), - ErrorGroups: m.migrateErrorGroups(&src.ErrorGroups, m.src.Server), - Admin: m.migrateAdmin(src.Admin, v1Cache), - // MassExport: m.migrateMassExport(src.MassExport, cfg), + ErrorGroups: m.migrateErrorGroups(&src.ErrorGroups, server.CH), + Admin: m.migrateAdmin(src.Admin, server.Cache), + MassExport: m.migrateMassExport(src.MassExport, &src.SeqAPI), } return dst } -func (m *v1tov2) migrateSeqAPI(src *v1.SeqAPI, v1Cache v1.Cache) *v2.SeqAPI { - dst := &v2.SeqAPI{ +func (m *v1tov2) migrateSeqAPI(src *v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { + dst := v2.SeqAPI{ Options: m.migrateSeqAPIOptions(src.SeqAPIOptions, v1Cache), DefaultEnv: src.DefaultEnv, } @@ -311,6 +318,8 @@ func (m *v1tov2) migrateSeqAPI(src *v1.SeqAPI, v1Cache v1.Cache) *v2.SeqAPI { dst.Envs = make(map[string]v2.SeqAPIEnv, len(src.Envs)) for envName, envCfg := range src.Envs { dstEnv := v2.SeqAPIEnv{SeqDBID: envCfg.SeqDB} + // in v1 env without options uses root options, in v2 root + // options are merged into env options if envCfg.Options != nil { options := m.migrateSeqAPIOptions(envCfg.Options, v1Cache) dstEnv.Options = &options @@ -354,6 +363,7 @@ func (m *v1tov2) migrateSeqAPIOptions(src *v1.SeqAPIOptions, v1Cache v1.Cache) v }, } + // in v1 fields are cached only if ttl is set if src.FieldsCacheTTL > 0 { dst.Caches.Fields = &v2.HandlerCache{ ID: v2.DefaultInmemID, @@ -436,19 +446,19 @@ func (m *v1tov2) migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet return dst } -func (m *v1tov2) migrateErrorGroups(eg *v1.ErrorGroups, server *v1.Server) *v2.ErrorGroups { - if server == nil || server.CH == nil { +func (m *v1tov2) migrateErrorGroups(src *v1.ErrorGroups, ch *v1.CH) *v2.ErrorGroups { + if ch == nil { return nil } return &v2.ErrorGroups{ Options: v2.ErrorGroupsOptions{ LogTagsMapping: v2.LogTagsMapping{ - Env: eg.LogTagsMapping.Env, - Service: eg.LogTagsMapping.Service, - Release: eg.LogTagsMapping.Release, + Env: src.LogTagsMapping.Env, + Service: src.LogTagsMapping.Service, + Release: src.LogTagsMapping.Release, }, - QueryFilter: eg.QueryFilter, + QueryFilter: src.QueryFilter, }, } } @@ -472,64 +482,61 @@ func (m *v1tov2) migrateAsyncSearch(src *v1.AsyncSearch, seqAPI *v1.SeqAPI) *v2. return dst } -// func (m *v1tov2) migrateMassExport(src *v1.MassExport, cfg *v2.Config) *v2.MassExport { -// if src == nil { -// return nil -// } -// -// dst := &v2.MassExport{ -// SeqDBID: v2.DefaultSeqDBClientID, -// Options: v2.MassExportOptions{ -// BatchSize: src.BatchSize, -// WorkersCount: src.WorkersCount, -// TasksChannelSize: src.TasksChannelSize, -// PartLength: src.PartLength, -// URLPrefix: src.URLPrefix, -// AllowedUsers: src.AllowedUsers, -// FileStore: m.migrateFileStore(src.FileStore), -// DownloadParams: m.migrateDownloadParams(m.src.SeqProxyDownloader), -// }, -// } -// -// if me.SessionStore != nil { -// cfg.Cache.Redis = append(cfg.Cache.Redis, m.migrateRedis(&src.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) -// -// dst.Options.SessionStore = v2.SessionStore{ -// RedisID: v2.DefaultMassExportRedisID, -// ExportLifetime: src.SessionStore.ExportLifetime, -// } -// } -// -// return dst -// } -// -// func (m *v1tov2) migrateFileStore(fs *v1.FileStore) v2.FileStore { -// if fs == nil || fs.S3 == nil { -// return v2.FileStore{} -// } -// -// return v2.FileStore{ -// S3: &v2.S3{ -// Endpoint: fs.S3.Endpoint, -// AccessKeyID: fs.S3.AccessKeyID, -// SecretAccessKey: fs.S3.SecretAccessKey, -// BucketName: fs.S3.BucketName, -// EnableSSL: fs.S3.EnableSSl, -// }, -// } -// } - -// func (m *v1tov2) migrateDownloadParams(src *v1.SeqProxyDownloader) v2.DownloadParams { -// if src == nil { -// return v2.DownloadParams{} -// } -// -// return v2.DownloadParams{ -// Delay: src.Delay, -// InitialRetryBackoff: src.InitialRetryBackoff, -// MaxRetryBackoff: src.MaxRetryBackoff, -// } -// } +func (m *v1tov2) migrateMassExport(src *v1.MassExport, seqAPI *v1.SeqAPI) *v2.MassExport { + if src == nil { + return nil + } + + options := v2.MassExportOptions{ + WorkersCount: src.WorkersCount, + TasksChannelSize: src.TasksChannelSize, + PartLength: src.PartLength, + URLPrefix: src.URLPrefix, + AllowedUsers: src.AllowedUsers, + DownloadParams: v2.DownloadParams{ + BatchSize: src.BatchSize, + }, + } + + if src.FileStore != nil && src.FileStore.S3 != nil { + options.FileStore.S3 = &v2.S3{ + Endpoint: src.FileStore.S3.Endpoint, + AccessKeyID: src.FileStore.S3.AccessKeyID, + SecretAccessKey: src.FileStore.S3.SecretAccessKey, + BucketName: src.FileStore.S3.BucketName, + EnableSSL: src.FileStore.S3.EnableSSl, + } + } + + if src.SessionStore != nil { + options.SessionStore = v2.SessionStore{ + Lifetime: src.SessionStore.ExportLifetime, + CacheID: v2.DefaultMassExportRedisID, + } + } + + if src.SeqProxyDownloader != nil { + options.DownloadParams.Delay = src.SeqProxyDownloader.Delay + options.DownloadParams.DelayBackoff = v2.Backoff{ + InitialInterval: src.SeqProxyDownloader.InitialRetryBackoff, + MaxInterval: src.SeqProxyDownloader.MaxRetryBackoff, + } + } + + dst := &v2.MassExport{Options: options} + + if len(seqAPI.Envs) > 0 { + dst.Envs = map[string]v2.MassExportEnv{ + seqAPI.DefaultEnv: { + SeqDBID: seqAPI.Envs[seqAPI.DefaultEnv].SeqDB, + }, + } + dst.DefaultEnv = seqAPI.DefaultEnv + } + + return dst +} + func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { if src == nil { return nil diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 9a26415..95a7d27 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -501,7 +501,7 @@ func (c *Clients) ClickHouseByID(id string) *CHClient { } type Handlers struct { - SeqAPI *SeqAPI `yaml:"seq_api"` + SeqAPI SeqAPI `yaml:"seq_api"` MassExport *MassExport `yaml:"mass_export"` AsyncSearch *AsyncSearch `yaml:"async_search"` ErrorGroups *ErrorGroups `yaml:"error_groups"` @@ -529,7 +529,6 @@ type Field struct { } type SeqAPI struct { - // добавить проверку на len(SEQDB) == 1 и берем [0] Options SeqAPIOptions `yaml:"options"` Envs map[string]SeqAPIEnv `yaml:"envs"` DefaultEnv string `yaml:"default_env"` diff --git a/internal/app/config/v2/validate.go b/internal/app/config/v2/validate.go index 023ff2a..aacc20b 100644 --- a/internal/app/config/v2/validate.go +++ b/internal/app/config/v2/validate.go @@ -3,6 +3,10 @@ package config import ( "fmt" "time" + + "go.uber.org/zap" + + "github.com/ozontech/seq-ui/logger" ) const ( @@ -13,6 +17,7 @@ const ( DefaultInmemID = "default_inmem" DefaultRedisWithInmemID = "default_redis_with_inmem" DefaultRedisWithoutInmemID = "default_redis_without_inmem" + DefaultMassExportRedisID = "default_mass_export_redis" SeqDBClientModeGRPC = "grpc" @@ -59,289 +64,211 @@ const ( defaultClickHouseReadTimeout = 30 * time.Second defaultUsePreparedStatements = true + + day = 24 * time.Hour + defaultSessionLifetime = 7 * day + minSessionLifetime = 1 * day + maxSessionLifetime = 30 * day ) var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE"} func (cfg *Config) Validate() error { - // seqDBIDs, chIDs, err := validateClients(&cfg.Clients) - // if err != nil { - // return err - // } - // cacheIDs, err := validateCache(&cfg.Cache) - // if err != nil { - // return err - // } + seqDBIDs, chIDs, err := validateClients(&cfg.Clients) + if err != nil { + return err + } + cacheIDs, err := validateCache(&cfg.Cache) + if err != nil { + return err + } validateDB(cfg.DB) - if err := validateServer(&cfg.Server); err != nil { + if err := validateServer(&cfg.Server, cacheIDs); err != nil { + return err + } + if err := validateSeqAPI(&cfg.Handlers.SeqAPI, seqDBIDs, cacheIDs); err != nil { + return err + } + if err := validateAsyncSearch(cfg.Handlers.AsyncSearch, seqDBIDs); err != nil { + return err + } + if err := validateMassExport(cfg.Handlers.MassExport, seqDBIDs, &cfg.Cache); err != nil { + return err + } + if err := validateErrorGroups(cfg.Handlers.ErrorGroups, chIDs); err != nil { + return err + } + if err := validateAdmin(cfg.Handlers.Admin, &cfg.Cache); err != nil { return err } - // if err := validateSeqAPI(cfg.Handlers.SeqAPI, seqDBIDs, cacheIDs); err != nil { - // return err - // } - // if err := validateAdmin(cfg.Handlers.Admin, &cfg.Cache); err != nil { - // return err - // } - - // if cfg.Handlers.MassExport != nil { - // me := cfg.Handlers.MassExport - // - // if len(me.Envs) > 0 { - // if me.SeqDBID != "" { - // return fmt.Errorf("handlers.mass_export.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") - // } - // if me.DefaultEnv == "" { - // return fmt.Errorf("handlers.mass_export.default_env must be specified when envs is used") - // } - // if _, exists := me.Envs[me.DefaultEnv]; !exists { - // return fmt.Errorf("handlers.mass_export.default_env %q not found in envs", me.DefaultEnv) - // } - // - // for envName, envConfig := range me.Envs { - // if envConfig.SeqDBID == "" { - // return fmt.Errorf("handlers.mass_export.envs[%q].seq_db_id cannot be empty", envName) - // } - // if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { - // return fmt.Errorf("unknown handlers.mass_export.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) - // } - // - // if envConfig.Options != nil { - // if envConfig.Options.SessionStore.RedisID == "" { - // return fmt.Errorf("handlers.mass_export.envs[%q].options.session_store.redis_id cannot be empty", envName) - // } - // if _, ok := redisIDs[envConfig.Options.SessionStore.RedisID]; !ok { - // return fmt.Errorf("unknown handlers.mass_export.envs[%q].options.session_store.redis_id %q", envName, envConfig.Options.SessionStore.RedisID) - // } - // if envConfig.Options.FileStore.S3 == nil { - // return fmt.Errorf("handlers.mass_export.envs[%q].options.file_store.s3 cannot be empty", envName) - // } - // } - // } - // } else { - // if me.SeqDBID == "" { - // return fmt.Errorf("handlers.mass_export.seq_db_id cannot be empty when envs is not used") - // } - // if _, ok := seqDBIDs[me.SeqDBID]; !ok { - // return fmt.Errorf("unknown handlers.mass_export.seq_db_id %q", me.SeqDBID) - // } - // if me.Options.SessionStore.RedisID == "" { - // return fmt.Errorf("handlers.mass_export.options.session_store.redis_id cannot be empty") - // } - // if _, ok := redisIDs[me.Options.SessionStore.RedisID]; !ok { - // return fmt.Errorf("unknown handlers.mass_export.options.session_store.redis_id %q", me.Options.SessionStore.RedisID) - // } - // } - // } - // - // if cfg.Handlers.AsyncSearch != nil { - // as := cfg.Handlers.AsyncSearch - // - // if as.Options.ListQueryLengthLimit <= 0 { - // as.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit - // } - // - // if len(as.Envs) > 0 { - // if as.SeqDBID != "" { - // return fmt.Errorf("handlers.async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") - // } - // if as.DefaultEnv == "" { - // return fmt.Errorf("handlers.async_search.default_env must be specified when envs is used") - // } - // if _, ok := as.Envs[as.DefaultEnv]; !ok { - // return fmt.Errorf("handlers.async_search.default_env %q not found in envs", as.DefaultEnv) - // } - // - // for envName, envConfig := range as.Envs { - // if envConfig.SeqDBID == "" { - // return fmt.Errorf("handlers.async_search.envs[%q].seq_db_id cannot be empty", envName) - // } - // if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { - // return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) - // } - // if envConfig.Options.ListQueryLengthLimit <= 0 { - // envConfig.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit - // } - // } - // } else { - // if as.SeqDBID == "" { - // return fmt.Errorf("handlers.async_search.seq_db_id cannot be empty when envs is not used") - // } - // if _, ok := seqDBIDs[as.SeqDBID]; !ok { - // return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", as.SeqDBID) - // } - // } - // } - // - // if cfg.Handlers.ErrorGroups != nil { - // eg := cfg.Handlers.ErrorGroups - // if len(eg.Envs) > 0 { - // if eg.CHID != "" { - // return fmt.Errorf("handlers.error_groups.ch_id must be empty when envs is used. Put ch_id inside each env") - // } - // if eg.DefaultEnv == "" { - // return fmt.Errorf("handlers.error_groups.default_env must be specified when envs is used") - // } - // if _, ok := eg.Envs[eg.DefaultEnv]; !ok { - // return fmt.Errorf("handlers.error_groups.default_env %q not found in envs", eg.DefaultEnv) - // } - // - // for envName, envConfig := range eg.Envs { - // if envConfig.CHID == "" { - // return fmt.Errorf("handlers.error_groups.envs[%q].ch_id cannot be empty", envName) - // } - // if _, ok := chIDs[envConfig.CHID]; !ok { - // return fmt.Errorf("unknown handlers.error_groups.envs[%q].ch_id %q", envName, envConfig.CHID) - // } - // } - // } else { - // if cfg.Handlers.ErrorGroups.CHID == "" { - // return fmt.Errorf("handlers.error_groups.ch_id cannot be empty when envs is not used") - // } - // if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { - // return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) - // } - // } - // } return nil } -func validateServer(server *Server) error { +func validateServer(server *Server, cacheIDs map[string]struct{}) error { if server.HTTP == nil && server.GRPC == nil { return fmt.Errorf("server: at least one of http or grpc must be set") } - if server.HTTP != nil && server.HTTP.CORS != nil { - if len(server.HTTP.CORS.AllowedOrigins) == 0 { - server.HTTP.CORS.AllowedOrigins = []string{defaultCORSAllowedOrigins} - } - if len(server.HTTP.CORS.AllowedMethods) == 0 { - server.HTTP.CORS.AllowedMethods = defaultCORSAllowedMethods - } + if server.HTTP != nil { + setCORSDefaults(server.HTTP.CORS) } + setCORSDefaults(server.Debug.CORS) if server.GRPC != nil && server.GRPC.ConnectionTimeout <= 0 { server.GRPC.ConnectionTimeout = defaultGRPCConnectionTimeout } + if server.Auth != nil { + if err := validateOIDC("server.auth.options.oidc", server.Auth.Options.OIDC, cacheIDs); err != nil { + return err + } + for envName, envCfg := range server.Auth.Envs { + if err := validateOIDC(fmt.Sprintf("server.auth.envs[%q].oidc", envName), envCfg.OIDC, cacheIDs); err != nil { + return err + } + } + } + + return nil +} + +func setCORSDefaults(cors *CORS) { + if cors == nil { + return + } + + if len(cors.AllowedOrigins) == 0 { + cors.AllowedOrigins = []string{defaultCORSAllowedOrigins} + } + if len(cors.AllowedMethods) == 0 { + cors.AllowedMethods = defaultCORSAllowedMethods + } +} + +func validateOIDC(path string, oidc *OIDC, cacheIDs map[string]struct{}) error { + if oidc == nil { + return nil + } + if len(oidc.AuthURLs) == 0 { + return fmt.Errorf("%s.auth_urls cannot be empty", path) + } + if (oidc.CacheID == "") != (oidc.CacheSecretKey == "") { + return fmt.Errorf("%s.cache_id and cache_secret_key must be set together", path) + } + if oidc.CacheID == "" { + return nil + } + if _, ok := cacheIDs[oidc.CacheID]; !ok { + return fmt.Errorf("%s.cache_id %q not found in cache", path, oidc.CacheID) + } return nil } -// func validateClients(clients *Clients) (seqDBIDs, chIDs map[string]struct{}, err error) { -// if len(clients.SeqDB) == 0 { -// return nil, nil, fmt.Errorf("clients.seq_db must contain at least one client") -// } -// -// seqDBIDs = make(map[string]struct{}, len(clients.SeqDB)) -// for i := range clients.SeqDB { -// c := &clients.SeqDB[i] -// if c.ID == "" { -// return nil, nil, fmt.Errorf("clients.seq_db[%d].id cannot be empty", i) -// } -// if _, ok := seqDBIDs[c.ID]; ok { -// return nil, nil, fmt.Errorf("duplicate clients.seq_db.id %q", c.ID) -// } -// -// seqDBIDs[c.ID] = struct{}{} -// -// if len(c.Addrs) == 0 { -// return nil, nil, fmt.Errorf("clients.seq_db[%q].addrs cannot be empty", c.ID) -// } -// if c.Timeout <= 0 { -// return nil, nil, fmt.Errorf("clients.seq_db[%q].timeout must be > 0", c.ID) -// } -// if c.ClientMode == "" { -// c.ClientMode = SeqDBClientModeGRPC -// } else if c.ClientMode != SeqDBClientModeGRPC { -// return nil, nil, fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, SeqDBClientModeGRPC) -// } -// -// if c.GRPCParams != nil { -// if c.GRPCParams.Keepalive != nil { -// c.GRPCParams.Keepalive.Time = max(c.GRPCParams.Keepalive.Time, minGRPCKeepaliveTime) -// c.GRPCParams.Keepalive.Timeout = max(c.GRPCParams.Keepalive.Timeout, minGRPCKeepaliveTimeout) -// } -// } -// } -// -// chIDs = make(map[string]struct{}, len(clients.ClickHouse)) -// for i := range clients.ClickHouse { -// ch := &clients.ClickHouse[i] -// if ch.ID == "" { -// return nil, nil, fmt.Errorf("clients.clickhouse[%d].id cannot be empty", i) -// } -// if _, ok := chIDs[ch.ID]; ok { -// return nil, nil, fmt.Errorf("duplicate clients.clickhouse.id %q", ch.ID) -// } -// -// chIDs[ch.ID] = struct{}{} -// -// if ch.DialTimeout <= 0 { -// ch.DialTimeout = defaultClickHouseDialTimeout -// } -// if ch.ReadTimeout <= 0 { -// ch.ReadTimeout = defaultClickHouseReadTimeout -// } -// } -// -// return seqDBIDs, chIDs, nil -// } -// -// func validateCache(cache *Cache) (map[string]struct{}, error) { -// inmemIDs := make(map[string]struct{}, len(cache.Inmemory)) -// for i := range cache.Inmemory { -// inm := &cache.Inmemory[i] -// if inm.ID == "" { -// return nil, fmt.Errorf("cache.inmemory[%d].id cannot be empty", i) -// } -// if _, ok := inmemIDs[inm.ID]; ok { -// return nil, fmt.Errorf("duplicate cache.inmemory.id %q", inm.ID) -// } -// -// inmemIDs[inm.ID] = struct{}{} -// -// if inm.NumCounters <= 0 { -// inm.NumCounters = defaultInmemCacheNumCounters -// } -// if inm.MaxCost <= 0 { -// inm.MaxCost = defaultInmemCacheMaxCost -// } -// if inm.BufferItems <= 0 { -// inm.BufferItems = defaultInmemCacheBufferItems -// } -// } -// -// cacheIDs := make(map[string]struct{}, len(cache.Inmemory)+len(cache.Redis)) -// for i := range cache.Redis { -// r := &cache.Redis[i] -// if r.ID == "" { -// return nil, fmt.Errorf("cache.redis[%d].id cannot be empty", i) -// } -// if _, ok := cacheIDs[r.ID]; ok { -// return nil, fmt.Errorf("duplicate cache.redis.id %q", r.ID) -// } -// -// cacheIDs[r.ID] = struct{}{} -// -// if r.WithInmemID != "" { -// if _, ok := inmemIDs[r.WithInmemID]; !ok { -// return nil, fmt.Errorf("cache.redis[%q].with_inmem_id %q not found in cache.inmemory", r.ID, r.WithInmemID) -// } -// } -// -// for id := range inmemIDs { -// if !slices.ContainsFunc(cache.Redis, func(r Redis) bool { -// return r.WithInmemID == id -// }) { -// cacheIDs[id] = struct{}{} -// } -// } -// } -// -// return cacheIDs, nil -// } +func validateClients(clients *Clients) (seqDBIDs, chIDs map[string]struct{}, err error) { + if len(clients.SeqDB) == 0 { + return nil, nil, fmt.Errorf("clients.seq_db must contain at least one client") + } + + seqDBIDs = make(map[string]struct{}, len(clients.SeqDB)) + for i := range clients.SeqDB { + c := &clients.SeqDB[i] + if c.ID == "" { + return nil, nil, fmt.Errorf("clients.seq_db[%d].id cannot be empty", i) + } + if _, ok := seqDBIDs[c.ID]; ok { + return nil, nil, fmt.Errorf("duplicate clients.seq_db.id %q", c.ID) + } + seqDBIDs[c.ID] = struct{}{} + + if len(c.Addrs) == 0 { + return nil, nil, fmt.Errorf("clients.seq_db[%q].addrs cannot be empty", c.ID) + } + if c.Timeout <= 0 { + return nil, nil, fmt.Errorf("clients.seq_db[%q].timeout must be > 0", c.ID) + } + if c.ClientMode == "" { + c.ClientMode = SeqDBClientModeGRPC + } else if c.ClientMode != SeqDBClientModeGRPC { + return nil, nil, fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, SeqDBClientModeGRPC) + } + + if c.GRPCParams != nil && c.GRPCParams.Keepalive != nil { + c.GRPCParams.Keepalive.Time = max(c.GRPCParams.Keepalive.Time, minGRPCKeepaliveTime) + c.GRPCParams.Keepalive.Timeout = max(c.GRPCParams.Keepalive.Timeout, minGRPCKeepaliveTimeout) + } + } + + chIDs = make(map[string]struct{}, len(clients.ClickHouse)) + for i := range clients.ClickHouse { + ch := &clients.ClickHouse[i] + if ch.ID == "" { + return nil, nil, fmt.Errorf("clients.clickhouse[%d].id cannot be empty", i) + } + if _, ok := chIDs[ch.ID]; ok { + return nil, nil, fmt.Errorf("duplicate clients.clickhouse.id %q", ch.ID) + } + chIDs[ch.ID] = struct{}{} + + if len(ch.Addrs) == 0 { + return nil, nil, fmt.Errorf("clients.clickhouse[%q].addrs cannot be empty", ch.ID) + } + if ch.DialTimeout <= 0 { + ch.DialTimeout = defaultClickHouseDialTimeout + } + if ch.ReadTimeout <= 0 { + ch.ReadTimeout = defaultClickHouseReadTimeout + } + } + + return seqDBIDs, chIDs, nil +} + +func validateCache(cache *Cache) (map[string]struct{}, error) { + cacheIDs := make(map[string]struct{}, len(cache.Inmemory)+len(cache.Redis)) + + for i := range cache.Inmemory { + inm := &cache.Inmemory[i] + if inm.ID == "" { + return nil, fmt.Errorf("cache.inmemory[%d].id cannot be empty", i) + } + if _, ok := cacheIDs[inm.ID]; ok { + return nil, fmt.Errorf("duplicate cache id %q", inm.ID) + } + cacheIDs[inm.ID] = struct{}{} + + if inm.NumCounters <= 0 { + inm.NumCounters = defaultInmemCacheNumCounters + } + if inm.MaxCost <= 0 { + inm.MaxCost = defaultInmemCacheMaxCost + } + if inm.BufferItems <= 0 { + inm.BufferItems = defaultInmemCacheBufferItems + } + } + + for i := range cache.Redis { + r := &cache.Redis[i] + if r.ID == "" { + return nil, fmt.Errorf("cache.redis[%d].id cannot be empty", i) + } + if _, ok := cacheIDs[r.ID]; ok { + return nil, fmt.Errorf("duplicate cache id %q", r.ID) + } + if r.Addr == "" { + return nil, fmt.Errorf("cache.redis[%q].addr cannot be empty", r.ID) + } + if r.WithInmemID != "" && cache.InmemByID(r.WithInmemID) == nil { + return nil, fmt.Errorf("cache.redis[%q].with_inmem_id %q not found in cache.inmemory", r.ID, r.WithInmemID) + } + cacheIDs[r.ID] = struct{}{} + } + + return cacheIDs, nil +} func validateDB(db *DB) { if db != nil && db.UsePreparedStatements == nil { @@ -350,100 +277,258 @@ func validateDB(db *DB) { } } -// func validateSeqAPI(seqAPI *SeqAPI, seqDBIDs, cacheIDs map[string]struct{}) error { -// setSeqAPIOptionsDefaults(&seqAPI.Options) -// -// _ = cacheIDs // Проверки потом нафигачу на кэши, пусть весит пока голенький -// if len(seqAPI.Envs) > 0 { -// if seqAPI.SeqDBID != "" { -// return fmt.Errorf("handlers.seq_api.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") -// } -// if seqAPI.DefaultEnv == "" { -// return fmt.Errorf("handlers.seq_api.default_env must be specified when envs is used") -// } -// if _, exists := seqAPI.Envs[seqAPI.DefaultEnv]; !exists { -// return fmt.Errorf("handlers.seq_api.default_env %q not found in envs", seqAPI.DefaultEnv) -// } -// -// for envName, envConfig := range seqAPI.Envs { -// if envConfig.SeqDBID == "" { -// return fmt.Errorf("handlers.seq_api.envs[%q].seq_db_id cannot be empty", envName) -// } -// if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { -// return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) -// } -// setSeqAPIOptionsDefaults(envConfig.Options) -// } -// } else { -// if seqAPI.SeqDBID == "" { -// return fmt.Errorf("handlers.seq_api.seq_db_id cannot be empty when envs is not used") -// } -// if _, ok := seqDBIDs[seqAPI.SeqDBID]; !ok { -// return fmt.Errorf("unknown handlers.seq_api.seq_db_id %q", seqAPI.SeqDBID) -// } -// } -// -// return nil -// } - -// func validateAdmin(admin *Admin, cache *Cache) error { -// if admin == nil { -// return nil -// } -// -// if admin.Options.Cache != nil { -// if admin.Options.Cache.ID == "" { -// return fmt.Errorf("handlers.admin.options.cache.id cannot be empty") -// } -// redisCfg := cache.RedisByID(admin.Options.Cache.ID) -// if redisCfg == nil { -// return fmt.Errorf("unknown handlers.admin.options.cache.id %q", admin.Options.Cache.ID) -// } -// if redisCfg == nil { -// return fmt.Errorf("unknown handlers.admin.options.cache.id %q", admin.Options.Cache.ID) -// } -// if redisCfg.WithInmemID != "" { -// return fmt.Errorf("handlers.admin.options.cache.id %q: with_inmem_id is not allowed", admin.Options.Cache.ID) -// } -// if admin.Options.Cache.TTL <= 0 { -// admin.Options.Cache.TTL = defaultAdminCacheTTL -// } -// } -// -// if len(admin.Envs) > 0 { -// -// } -// } -// -// func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { -// if options.Limits.MaxAggregationsPerRequest <= 0 { -// options.Limits.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest -// } -// if options.Limits.MaxBucketsPerAggregationTs <= 0 { -// options.Limits.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs -// } -// if options.Limits.MaxParallelExportRequests <= 0 { -// options.Limits.MaxParallelExportRequests = defaultMaxParallelExportRequests -// } -// if options.Limits.MaxSearchTotal <= 0 { -// options.Limits.MaxSearchTotal = defaultMaxSearchTotal -// } -// if options.Limits.MaxSearchOffset <= 0 { -// options.Limits.MaxSearchOffset = defaultMaxSearchOffset -// } -// if options.Limits.MaxExportLimit <= 0 { -// options.Limits.MaxExportLimit = defaultMaxExportLimit -// } -// if options.Limits.MaxSearchLimit <= 0 { -// options.Limits.MaxSearchLimit = defaultMaxSearchLimit -// } -// if options.Limits.SeqCLIMaxSearchLimit <= 0 { -// options.Limits.SeqCLIMaxSearchLimit = defaultSeqCLIMaxSearchLimit -// } -// if options.Caches.Events.TTL <= 0 { -// options.Caches.Events.TTL = defaultEventsCacheTTL -// } -// if options.Caches.LogsLifespan.TTL <= 0 { -// options.Caches.LogsLifespan.TTL = defaultLogsLifespanCacheTTL -// } -// } +func validateSeqAPI(seqAPI *SeqAPI, seqDBIDs, cacheIDs map[string]struct{}) error { + if err := validateEnvs("handlers.seq_api", seqAPI.Envs, seqAPI.DefaultEnv); err != nil { + return err + } + if len(seqAPI.Envs) == 0 && len(seqDBIDs) != 1 { + return fmt.Errorf("handlers.seq_api: clients.seq_db must contain one client when envs are not used") + } + if err := validateSeqAPIOptions("handlers.seq_api.options", &seqAPI.Options, cacheIDs); err != nil { + return err + } + + for envName, envConfig := range seqAPI.Envs { + if envConfig.SeqDBID == "" { + return fmt.Errorf("handlers.seq_api.envs[%q].seq_db_id cannot be empty", envName) + } + if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + } + + // options are nil only if root options are not set, so env uses default options + if envConfig.Options == nil { + envConfig.Options = &SeqAPIOptions{} + seqAPI.Envs[envName] = envConfig + } + if err := validateSeqAPIOptions(fmt.Sprintf("handlers.seq_api.envs[%q].options", envName), envConfig.Options, cacheIDs); err != nil { + return err + } + } + + return nil +} + +func validateAdmin(admin *Admin, cache *Cache) error { + if admin == nil { + return nil + } + if err := validateEnvs("handlers.admin", admin.Envs, admin.DefaultEnv); err != nil { + return err + } + if err := validateAdminOptions("handlers.admin.options", &admin.Options, cache); err != nil { + return err + } + + for envName, envConfig := range admin.Envs { + if envConfig.Options == nil { + envConfig.Options = &AdminOptions{} + admin.Envs[envName] = envConfig + } + if err := validateAdminOptions(fmt.Sprintf("handlers.admin.envs[%q].options", envName), envConfig.Options, cache); err != nil { + return err + } + } + + return nil +} + +func validateAdminOptions(path string, options *AdminOptions, cache *Cache) error { + if options.Cache == nil { + return nil + } + + redis := cache.RedisByID(options.Cache.ID) + if redis == nil || redis.WithInmemID != "" { + return fmt.Errorf("%s.cache_id %q must be cache.redis without with_inmem_id", path, options.Cache.ID) + } + if options.Cache.TTL <= 0 { + options.Cache.TTL = defaultAdminCacheTTL + } + + return nil +} + +func validateEnvs[T any](path string, envs map[string]T, defaultEnv string) error { + if len(envs) == 0 { + return nil + } + if defaultEnv == "" { + return fmt.Errorf("%s.default_env must be set when envs are used", path) + } + if _, ok := envs[defaultEnv]; !ok { + return fmt.Errorf("%s.default_env %q not found in envs", path, defaultEnv) + } + return nil +} + +func validateAsyncSearch(asyncSearch *AsyncSearch, seqDBIDs map[string]struct{}) error { + if asyncSearch == nil { + return nil + } + if err := validateEnvs("handlers.async_search", asyncSearch.Envs, asyncSearch.DefaultEnv); err != nil { + return err + } + if asyncSearch.Options.ListQueryLengthLimit <= 0 { + asyncSearch.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + } + + for envName, envConfig := range asyncSearch.Envs { + if envConfig.SeqDBID == "" { + return fmt.Errorf("handlers.async_search.envs[%q].seq_db_id cannot be empty", envName) + } + if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + } + if envConfig.Options == nil { + envConfig.Options = &AsyncSearchOptions{} + asyncSearch.Envs[envName] = envConfig + } + if envConfig.Options.ListQueryLengthLimit <= 0 { + envConfig.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + } + } + + return nil +} + +func validateSeqAPIOptions(path string, options *SeqAPIOptions, cacheIDs map[string]struct{}) error { + limits := &options.Limits + if limits.MaxAggregationsPerRequest <= 0 { + limits.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest + } + if limits.MaxBucketsPerAggregationTs <= 0 { + limits.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs + } + if limits.MaxParallelExportRequests <= 0 { + limits.MaxParallelExportRequests = defaultMaxParallelExportRequests + } + if limits.MaxSearchTotal <= 0 { + limits.MaxSearchTotal = defaultMaxSearchTotal + } + if limits.MaxSearchOffset <= 0 { + limits.MaxSearchOffset = defaultMaxSearchOffset + } + if limits.MaxExportLimit <= 0 { + limits.MaxExportLimit = defaultMaxExportLimit + } + if limits.MaxSearchLimit <= 0 { + limits.MaxSearchLimit = defaultMaxSearchLimit + } + if limits.SeqCLIMaxSearchLimit <= 0 { + limits.SeqCLIMaxSearchLimit = defaultSeqCLIMaxSearchLimit + } + + caches := &options.Caches + if err := validateHandlerCache(path+".caches.events", caches.Events, cacheIDs, defaultEventsCacheTTL); err != nil { + return err + } + if err := validateHandlerCache(path+".caches.logs_lifespan", caches.LogsLifespan, cacheIDs, defaultLogsLifespanCacheTTL); err != nil { + return err + } + + return validateHandlerCache(path+".caches.fields", caches.Fields, cacheIDs, 0) +} + +func validateHandlerCache(path string, hc *HandlerCache, cacheIDs map[string]struct{}, defaultTTL time.Duration) error { + if hc == nil { + return nil + } + if _, ok := cacheIDs[hc.ID]; !ok { + return fmt.Errorf("%s.id %q not found in cache", path, hc.ID) + } + if hc.TTL <= 0 { + hc.TTL = defaultTTL + } + + return nil +} + +func validateErrorGroups(errorGroups *ErrorGroups, chIDs map[string]struct{}) error { + if errorGroups == nil { + return nil + } + + if err := validateEnvs("handlers.error_groups", errorGroups.Envs, errorGroups.DefaultEnv); err != nil { + return err + } + + for envName, envConfig := range errorGroups.Envs { + if envConfig.CHID == "" { + return fmt.Errorf("handlers.error_groups.envs[%q].ch_id cannot be empty", envName) + } + if _, ok := chIDs[envConfig.CHID]; !ok { + return fmt.Errorf("unknown handlers.error_groups.envs[%q].ch_id %q", envName, envConfig.CHID) + } + if envConfig.Options == nil { + envConfig.Options = &ErrorGroupsOptions{} + errorGroups.Envs[envName] = envConfig + } + } + + return nil +} + +func validateMassExport(massExport *MassExport, seqDBIDs map[string]struct{}, cache *Cache) error { + if massExport == nil { + return nil + } + if err := validateEnvs("handlers.mass_export", massExport.Envs, massExport.DefaultEnv); err != nil { + return err + } + if len(massExport.Envs) == 0 { + return validateMassExportOptions("handlers.mass_export.options", &massExport.Options, cache, "") + } + + for envName, envConfig := range massExport.Envs { + if envConfig.SeqDBID == "" { + return fmt.Errorf("handlers.mass_export.envs[%q].seq_db_id cannot be empty", envName) + } + if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.mass_export.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + } + if envConfig.Options == nil { + envConfig.Options = &MassExportOptions{} + massExport.Envs[envName] = envConfig + } + if err := validateMassExportOptions(fmt.Sprintf("handlers.mass_export.envs[%q].options", envName), envConfig.Options, cache, envName); err != nil { + return err + } + } + + return nil +} + +func validateMassExportOptions(path string, options *MassExportOptions, cache *Cache, env string) error { + if options.FileStore.S3 == nil { + return fmt.Errorf("%s.file_store.s3 cannot be empty", path) + } + + redis := cache.RedisByID(options.SessionStore.CacheID) + if redis == nil || redis.WithInmemID != "" { + return fmt.Errorf("%s.session_store.cache_id %q must be cache.redis without with_inmem_id", path, options.SessionStore.CacheID) + } + + setSessionStoreDefaults(&options.SessionStore, env) + + return nil +} + +func setSessionStoreDefaults(sessionStore *SessionStore, env string) { + switch { + case sessionStore.Lifetime == 0: + sessionStore.Lifetime = defaultSessionLifetime + case sessionStore.Lifetime < minSessionLifetime: + logger.Warn("export lifetime from config is too low; set min lifetime", + zap.String("env", env), + zap.String("export_lifetime", sessionStore.Lifetime.String()), + ) + sessionStore.Lifetime = minSessionLifetime + case sessionStore.Lifetime > maxSessionLifetime: + logger.Warn("export lifetime from config is too high; set max lifetime", + zap.String("env", env), + zap.String("export_lifetime", sessionStore.Lifetime.String()), + ) + sessionStore.Lifetime = maxSessionLifetime + } +} diff --git a/internal/app/mw/env.go b/internal/app/mw/env.go index 1f15ba8..f66143e 100644 --- a/internal/app/mw/env.go +++ b/internal/app/mw/env.go @@ -21,10 +21,9 @@ type EnvResolver map[string]handlerEnvs func NewEnvResolver(handlers *config.Handlers) EnvResolver { r := EnvResolver{} - if h := handlers.SeqAPI; h != nil { - envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} - r["seqapi"], r["SeqAPIService"] = envs, envs - } + seqAPIEnvs := handlerEnvs{DefaultEnv: handlers.SeqAPI.DefaultEnv, Envs: envNames(handlers.SeqAPI.Envs)} + r["seqapi"], r["SeqAPIService"] = seqAPIEnvs, seqAPIEnvs + if h := handlers.ErrorGroups; h != nil { envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} r["errorgroups"], r["ErrorGroupsService"] = envs, envs diff --git a/internal/pkg/cache/redis.go b/internal/pkg/cache/redis.go index 2676592..d7e5b11 100644 --- a/internal/pkg/cache/redis.go +++ b/internal/pkg/cache/redis.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/redis/go-redis/v9" @@ -13,6 +14,8 @@ import ( "github.com/ozontech/seq-ui/metric" ) +const keysBatchSize = 10000 + type redisCache struct { client *redis.Client prefix string @@ -75,6 +78,44 @@ func (c *redisCache) Del(ctx context.Context, key string) { c.client.Del(ctx, c.withPrefix(key)) } +func (c *redisCache) SetNX(ctx context.Context, key string, value string, ttl time.Duration) (bool, error) { + return c.client.SetNX(ctx, c.withPrefix(key), value, ttl).Result() +} + +func (c *redisCache) KeysByPattern(ctx context.Context, pattern string) ([]string, error) { + var ( + cursor uint64 + result, partRes []string + err error + ) + + for { + partRes, cursor, err = c.client.Scan(ctx, cursor, c.withPrefix(pattern), keysBatchSize).Result() + if err != nil { + return nil, fmt.Errorf("redis scan: %w", err) + } + + for _, key := range partRes { + result = append(result, strings.TrimPrefix(key, c.prefix)) + } + + if cursor == 0 { + break + } + } + + return result, nil +} + +func (c *redisCache) MGet(ctx context.Context, keys ...string) (any, error) { + prefixedKeys := make([]string, len(keys)) + for i, key := range keys { + prefixedKeys[i] = c.withPrefix(key) + } + + return c.client.MGet(ctx, keys...).Result() +} + func (c *redisCache) withPrefix(key string) string { return c.prefix + key } diff --git a/internal/pkg/cache/store.go b/internal/pkg/cache/store.go new file mode 100644 index 0000000..55c24d5 --- /dev/null +++ b/internal/pkg/cache/store.go @@ -0,0 +1,18 @@ +package cache + +import ( + "context" + "time" +) + +type Store interface { + Cache + SetNX(ctx context.Context, key string, value string, ttl time.Duration) (bool, error) + KeysByPattern(ctx context.Context, pattern string) ([]string, error) + MGet(ctx context.Context, keys ...string) ([]any, error) +} + +func StoreByID(id string) (Store, bool) { + store, ok := caches[id].(Store) + return store, ok +} diff --git a/internal/pkg/service/admin/mock/service.go b/internal/pkg/service/admin/mock/service.go index ffcad8c..42a760f 100644 --- a/internal/pkg/service/admin/mock/service.go +++ b/internal/pkg/service/admin/mock/service.go @@ -98,20 +98,6 @@ func (mr *MockServiceMockRecorder) DeleteUsersFromRole(arg0, arg1 any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteUsersFromRole", reflect.TypeOf((*MockService)(nil).DeleteUsersFromRole), arg0, arg1) } -// GetAvailablePermissions mocks base method. -func (m *MockService) GetAvailablePermissions() []types.PermissionGroup { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAvailablePermissions") - ret0, _ := ret[0].([]types.PermissionGroup) - return ret0 -} - -// GetAvailablePermissions indicates an expected call of GetAvailablePermissions. -func (mr *MockServiceMockRecorder) GetAvailablePermissions() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAvailablePermissions", reflect.TypeOf((*MockService)(nil).GetAvailablePermissions)) -} - // GetRole mocks base method. func (m *MockService) GetRole(arg0 context.Context, arg1 types.GetRoleRequest) (types.RoleInfo, error) { m.ctrl.T.Helper() diff --git a/internal/pkg/service/admin/permissions.go b/internal/pkg/service/admin/permissions.go index 3d27db2..e304fe2 100644 --- a/internal/pkg/service/admin/permissions.go +++ b/internal/pkg/service/admin/permissions.go @@ -15,7 +15,7 @@ var availablePermissions = []string{ permissionRolesDelete, } -func (s *service) GetAvailablePermissions() []types.PermissionGroup { +func GetAvailablePermissions() []types.PermissionGroup { return []types.PermissionGroup{ { Group: "roles", diff --git a/internal/pkg/service/admin/permissions_test.go b/internal/pkg/service/admin/permissions_test.go index 5294654..898ab4d 100644 --- a/internal/pkg/service/admin/permissions_test.go +++ b/internal/pkg/service/admin/permissions_test.go @@ -68,7 +68,8 @@ func TestCheckAccess(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "").(*service) + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") if tt.mockArgs != nil { cache.EXPECT(). @@ -152,7 +153,8 @@ func TestGetUserPermissions(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "").(*service) + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") cache.EXPECT(). Get(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username). @@ -220,7 +222,8 @@ func TestValidatePermissions(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "").(*service) + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") err := svc.validatePermissions(tt.perms) require.Equal(t, tt.wantErr, err != nil) diff --git a/internal/pkg/service/admin/roles_test.go b/internal/pkg/service/admin/roles_test.go index babfcf5..f8284ac 100644 --- a/internal/pkg/service/admin/roles_test.go +++ b/internal/pkg/service/admin/roles_test.go @@ -198,7 +198,8 @@ func TestCreateRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "") + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -345,7 +346,8 @@ func TestAddUsersToRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "") + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -463,7 +465,8 @@ func TestGetRoles(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "") + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -575,7 +578,8 @@ func TestGetRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "") + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -786,7 +790,8 @@ func TestUpdateRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "") + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -944,7 +949,8 @@ func TestDeleteRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "") + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -1100,7 +1106,8 @@ func TestDeleteUsersFromRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg, "") + svc := newService(adminRepo, upRepo, adminCfg, "").(*service) + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) diff --git a/internal/pkg/service/admin/service.go b/internal/pkg/service/admin/service.go index 9d7d9e8..25211ad 100644 --- a/internal/pkg/service/admin/service.go +++ b/internal/pkg/service/admin/service.go @@ -18,7 +18,6 @@ type Service interface { GetRole(context.Context, types.GetRoleRequest) (types.RoleInfo, error) UpdateRole(context.Context, types.UpdateRoleRequest) error DeleteRole(context.Context, types.DeleteRoleRequest) error - GetAvailablePermissions() []types.PermissionGroup } type service struct { @@ -29,14 +28,31 @@ type service struct { env string } -func New(adminRepo repository.Admin, userProfilesRepo repository.UserProfiles, c cache.Cache, cfg *config.AdminOptions, env string) Service { +func New(cfg *config.Admin, adminRepo repository.Admin, userProfilesRepo repository.UserProfiles) map[string]Service { + serviceByEnv := make(map[string]Service, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envCfg := range cfg.Envs { + serviceByEnv[envName] = newService(adminRepo, userProfilesRepo, envCfg.Options, envName) + } + } else { + serviceByEnv[""] = newService(adminRepo, userProfilesRepo, &cfg.Options, "") + } + + return serviceByEnv +} + +func newService(adminRepo repository.Admin, userProfilesRepo repository.UserProfiles, cfg *config.AdminOptions, env string) Service { su := make(map[string]struct{}, len(cfg.SuperUsers)) for _, u := range cfg.SuperUsers { su[u] = struct{}{} } - var ttl time.Duration + var ( + c cache.Cache + ttl time.Duration + ) if cfg.Cache != nil { + c, _ = cache.ByID(cfg.Cache.ID) ttl = cfg.Cache.TTL } @@ -48,24 +64,3 @@ func New(adminRepo repository.Admin, userProfilesRepo repository.UserProfiles, c env: env, } } - -func NewByEnv(cfg *config.Admin, adminRepo repository.Admin, userProfilesRepo repository.UserProfiles) map[string]Service { - services := make(map[string]Service, max(1, len(cfg.Envs))) - if len(cfg.Envs) > 0 { - for envName, envCfg := range cfg.Envs { - var c cache.Cache - if envCfg.Options.Cache != nil { - c, _ = cache.ByID(envCfg.Options.Cache.ID) - } - services[envName] = New(adminRepo, userProfilesRepo, c, envCfg.Options, envName) - } - } else { - var c cache.Cache - if cfg.Options.Cache != nil { - c, _ = cache.ByID(cfg.Options.Cache.ID) - } - services[""] = New(adminRepo, userProfilesRepo, c, &cfg.Options, "") - } - - return services -} diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index 8af7bb0..38f44c0 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -3,6 +3,7 @@ package asyncsearches import ( "context" "fmt" + "maps" "slices" "time" "unicode/utf8" @@ -45,46 +46,38 @@ type service struct { env string } -func New(ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, cfg *config.AsyncSearchOptions, env string) Service { - s := &service{ - repo: repo, - seqDB: seqDB, - cfg: cfg, - env: env, - } - - go s.deleteExpiredAsyncSearches(ctx) - - return s -} - -// NewByEnv creates service for each env. -func NewByEnv( +func New( ctx context.Context, cfg *config.AsyncSearch, repo repository.AsyncSearches, seqDBClients map[string]seqdb.Client, ) map[string]Service { - if cfg == nil || repo == nil { - return nil - } - services := make(map[string]Service, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { for envName, envCfg := range cfg.Envs { - services[envName] = New(ctx, repo, seqDBClients[envCfg.SeqDBID], envCfg.Options, envName) + services[envName] = newService(ctx, repo, seqDBClients[envCfg.SeqDBID], envCfg.Options, envName) } } else { - var client seqdb.Client - for _, c := range seqDBClients { - client = c - } - services[""] = New(ctx, repo, client, &cfg.Options, "") + client := slices.Collect(maps.Values(seqDBClients))[0] + services[""] = newService(ctx, repo, client, &cfg.Options, "") } return services } +func newService(ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, cfg *config.AsyncSearchOptions, env string) Service { + s := &service{ + repo: repo, + seqDB: seqDB, + cfg: cfg, + env: env, + } + + go s.deleteExpiredAsyncSearches(ctx) + + return s +} + func (s *service) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSearchRequest) (*seqapi.StartAsyncSearchResponse, error) { ownerID, err := profiles.GetIDFromContext(ctx) if err != nil { diff --git a/internal/pkg/service/errorgroups/service.go b/internal/pkg/service/errorgroups/service.go index d7fd96e..e21c5f7 100644 --- a/internal/pkg/service/errorgroups/service.go +++ b/internal/pkg/service/errorgroups/service.go @@ -40,32 +40,29 @@ type service struct { logTagsMapping config.LogTagsMapping } -func New(repo repositorych.Repository, logTagsMapping config.LogTagsMapping) Service { - return &service{ - repo: repo, - logTagsMapping: logTagsMapping, - } -} - -func NewByEnv(cfg *config.ErrorGroups, clientsCfg *config.Clients, chClients map[string]driver.Conn) map[string]Service { +func New(cfg *config.ErrorGroups, clientsCfg *config.Clients, chClients map[string]driver.Conn) map[string]Service { services := make(map[string]Service, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { for envName, envCfg := range cfg.Envs { repo := repositorych.New(chClients[envCfg.CHID], clientsCfg.ClickHouseByID(envCfg.CHID).Sharded, envCfg.Options.QueryFilter) - services[envName] = New(repo, envCfg.Options.LogTagsMapping) + services[envName] = newService(repo, envCfg.Options.LogTagsMapping) } } else { - var chID string - for id := range chClients { - chID = id - } + chID := slices.Collect(maps.Keys(chClients))[0] repo := repositorych.New(chClients[chID], clientsCfg.ClickHouseByID(chID).Sharded, cfg.Options.QueryFilter) - services[""] = New(repo, cfg.Options.LogTagsMapping) + services[""] = newService(repo, cfg.Options.LogTagsMapping) } return services } +func newService(repo repositorych.Repository, logTagsMapping config.LogTagsMapping) Service { + return &service{ + repo: repo, + logTagsMapping: logTagsMapping, + } +} + func (s *service) GetErrorGroups( ctx context.Context, req types.GetErrorGroupsRequest, diff --git a/internal/pkg/service/errorgroups/service_test.go b/internal/pkg/service/errorgroups/service_test.go index b9bd1ab..ec18f1c 100644 --- a/internal/pkg/service/errorgroups/service_test.go +++ b/internal/pkg/service/errorgroups/service_test.go @@ -283,7 +283,7 @@ func TestGetErrorGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedRepo := mock.NewMockRepository(ctrl) - svc := New(mockedRepo, config.LogTagsMapping{}) + svc := newService(mockedRepo, config.LogTagsMapping{}) if ma := tt.mockArgs; ma != nil { mockedRepo.EXPECT(). @@ -514,7 +514,7 @@ func TestGetNewErrorGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedRepo := mock.NewMockRepository(ctrl) - svc := New(mockedRepo, config.LogTagsMapping{}) + svc := newService(mockedRepo, config.LogTagsMapping{}) if ma := tt.mockArgs; ma != nil { mockedRepo.EXPECT(). @@ -701,7 +701,7 @@ func TestGetTopErrorGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedRepo := mock.NewMockRepository(ctrl) - svc := New(mockedRepo, config.LogTagsMapping{}) + svc := newService(mockedRepo, config.LogTagsMapping{}) if ma := tt.mockArgs; ma != nil { mockedRepo.EXPECT(). @@ -816,7 +816,7 @@ func TestGetHist(t *testing.T) { ctrl := gomock.NewController(t) mockedRepo := mock.NewMockRepository(ctrl) - svc := New(mockedRepo, config.LogTagsMapping{}) + svc := newService(mockedRepo, config.LogTagsMapping{}) if ma := tt.mockArgs; ma != nil { mockedRepo.EXPECT(). @@ -1150,7 +1150,7 @@ func TestGetDetails(t *testing.T) { ctrl := gomock.NewController(t) mockedRepo := mock.NewMockRepository(ctrl) - svc := New(mockedRepo, tt.logTagsMapping) + svc := newService(mockedRepo, tt.logTagsMapping) if ma := tt.mockArgs; ma != nil { mockedRepo.EXPECT(). @@ -1228,7 +1228,7 @@ func TestGetServices(t *testing.T) { ctrl := gomock.NewController(t) mockedRepo := mock.NewMockRepository(ctrl) - svc := New(mockedRepo, config.LogTagsMapping{}) + svc := newService(mockedRepo, config.LogTagsMapping{}) if ma := tt.mockArgs; ma != nil { mockedRepo.EXPECT(). @@ -1318,7 +1318,7 @@ func TestGetReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedRepo := mock.NewMockRepository(ctrl) - svc := New(mockedRepo, config.LogTagsMapping{}) + svc := newService(mockedRepo, config.LogTagsMapping{}) if ma := tt.mockArgs; ma != nil { mockedRepo.EXPECT(). @@ -1510,7 +1510,7 @@ func TestDiffByReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedRepo := mock.NewMockRepository(ctrl) - svc := New(mockedRepo, config.LogTagsMapping{}) + svc := newService(mockedRepo, config.LogTagsMapping{}) if ma := tt.mockArgs; ma != nil { mockedRepo.EXPECT(). @@ -1617,7 +1617,7 @@ func TestGetServiceFilters(t *testing.T) { ctrl := gomock.NewController(t) mockedRepo := mock.NewMockRepository(ctrl) - svc := New(mockedRepo, config.LogTagsMapping{}) + svc := newService(mockedRepo, config.LogTagsMapping{}) if ma := tt.mockArgs; ma != nil { mockedRepo.EXPECT(). diff --git a/internal/pkg/service/massexport/seq_proxy_downloader.go b/internal/pkg/service/massexport/seq_proxy_downloader.go index 5823e14..f5252f3 100644 --- a/internal/pkg/service/massexport/seq_proxy_downloader.go +++ b/internal/pkg/service/massexport/seq_proxy_downloader.go @@ -28,8 +28,8 @@ func newSeqProxyDownloader(client seqdb.Client, cfg config.DownloadParams) *seqP cfg.Delay = defaultDelay } - cfg.InitialRetryBackoff = max(cfg.InitialRetryBackoff, cfg.Delay) - cfg.MaxRetryBackoff = max(cfg.MaxRetryBackoff, cfg.InitialRetryBackoff) + cfg.DelayBackoff.InitialInterval = max(cfg.DelayBackoff.InitialInterval, cfg.Delay) + cfg.DelayBackoff.MaxInterval = max(cfg.DelayBackoff.MaxInterval, cfg.DelayBackoff.InitialInterval) return &seqProxyDownloader{ cfg: cfg, @@ -41,8 +41,8 @@ func (d *seqProxyDownloader) Search(ctx context.Context, sessionID string, req * bo := backoff.NewExponentialBackOff() bo.MaxElapsedTime = 0 bo.Multiplier = 2 - bo.InitialInterval = d.cfg.InitialRetryBackoff - bo.MaxInterval = d.cfg.MaxRetryBackoff + bo.InitialInterval = d.cfg.DelayBackoff.InitialInterval + bo.MaxInterval = d.cfg.DelayBackoff.MaxInterval bo.Reset() for { diff --git a/internal/pkg/service/massexport/service.go b/internal/pkg/service/massexport/service.go index ee62510..72e8223 100644 --- a/internal/pkg/service/massexport/service.go +++ b/internal/pkg/service/massexport/service.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "maps" "slices" "time" @@ -50,16 +51,44 @@ const ( defaultPartLength = 1 * time.Hour ) -func NewService( - ctx context.Context, - cfg config.MassExport, - sessionStore sessionstore.SessionStore, - fileStore filestore.FileStore, - client seqdb.Client, -) (Service, error) { +func New(ctx context.Context, cfg *config.MassExport, seqDBClients map[string]seqdb.Client) (map[string]Service, error) { + services := make(map[string]Service, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envCfg := range cfg.Envs { + svc, err := newService(ctx, envCfg.Options, seqDBClients[envCfg.SeqDBID]) + if err != nil { + return nil, fmt.Errorf("env %q: %w", envName, err) + } + services[envName] = svc + } + } else { + client := slices.Collect(maps.Values(seqDBClients))[0] + svc, err := newService(ctx, &cfg.Options, client) + if err != nil { + return nil, err + } + services[""] = svc + } + + return services, nil +} + +func newService(ctx context.Context, cfg *config.MassExportOptions, client seqdb.Client) (Service, error) { + sessionStore, err := sessionstore.NewRedisSessionStore(ctx, &cfg.SessionStore) + if err != nil { + return nil, fmt.Errorf("init session store: %w", err) + } + logger.Info("session store initialized") + + fileStore, err := filestore.NewS3(ctx, cfg.FileStore.S3) + if err != nil { + return nil, fmt.Errorf("init file store: %w", err) + } + logger.Info("file store initialized") + batchSize := defaultBatchSize - if cfg.BatchSize > 0 { - batchSize = cfg.BatchSize + if cfg.DownloadParams.BatchSize > 0 { + batchSize = cfg.DownloadParams.BatchSize } if cfg.WorkersCount <= 0 { @@ -68,7 +97,7 @@ func NewService( partLength := defaultPartLength if cfg.PartLength > 0 { - err := checkPartLength(cfg.PartLength) + err = checkPartLength(cfg.PartLength) if err != nil { return nil, fmt.Errorf("check part length: %w", err) } @@ -93,7 +122,7 @@ func NewService( tasksChannelSize = cfg.TasksChannelSize } - downloader := newSeqProxyDownloader(client, *cfg.DownloadParams) + downloader := newSeqProxyDownloader(client, cfg.DownloadParams) return &exportService{ sessionStore: sessionStore, diff --git a/internal/pkg/service/massexport/sessionstore/redis.go b/internal/pkg/service/massexport/sessionstore/redis.go index c4b245f..2cfe524 100644 --- a/internal/pkg/service/massexport/sessionstore/redis.go +++ b/internal/pkg/service/massexport/sessionstore/redis.go @@ -8,63 +8,41 @@ import ( "strings" "time" - "github.com/redis/go-redis/v9" "go.uber.org/zap" "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/internal/pkg/redisclient" + "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" ) -type redisSessionStore struct { - client *redis.Client - exportLifetime time.Duration +type sessionStore struct { + client cache.Store + lifetime time.Duration } const ( activeSessionIDKey = "active_session_id" - - day = 24 * time.Hour - defaultLifetime = 7 * day - minLifetime = 1 * day - maxLifetime = 30 * day + exportsKeysPattern = "job#*#export#*" ) -func NewRedisSessionStore(ctx context.Context, redisCfg *config.Redis, exportLifetimeCfg time.Duration) (SessionStore, error) { - client, err := redisclient.New(ctx, redisCfg) - if err != nil { - return nil, fmt.Errorf("create redis client: %w", err) - } - - exportLifetime := defaultLifetime - if exportLifetimeCfg != 0 { - switch { - case exportLifetimeCfg < minLifetime: - logger.Warn("export lifetime from config is too low; set min lifetime", zap.String("export_lifetime", exportLifetimeCfg.String())) - exportLifetime = minLifetime - case exportLifetimeCfg > maxLifetime: - logger.Warn("export lifetime from config is too high; set max lifetime", zap.String("export_lifetime", exportLifetimeCfg.String())) - exportLifetime = maxLifetime - default: - exportLifetime = exportLifetimeCfg - } +func NewRedisSessionStore(ctx context.Context, cfg *config.SessionStore) (SessionStore, error) { + client, ok := cache.StoreByID(cfg.CacheID) + if !ok { + return nil, fmt.Errorf("cache %q not found", cfg.CacheID) } - store := &redisSessionStore{ - client: client, - exportLifetime: exportLifetime, + store := &sessionStore{ + client: client, + lifetime: cfg.Lifetime, } - err = store.unlock(ctx) - if err != nil { - return nil, err - } + store.unlock(ctx) return store, nil } -func (s *redisSessionStore) StartExport(ctx context.Context, sessionID string, info types.ExportInfo) error { +func (s *sessionStore) StartExport(ctx context.Context, sessionID string, info types.ExportInfo) error { err := s.Lock(ctx, sessionID) if err != nil { return err @@ -73,121 +51,49 @@ func (s *redisSessionStore) StartExport(ctx context.Context, sessionID string, i return s.set(ctx, sessionID, info) } -func (s *redisSessionStore) CheckExport(ctx context.Context, sessionID string) (types.ExportInfo, error) { +func (s *sessionStore) CheckExport(ctx context.Context, sessionID string) (types.ExportInfo, error) { return s.get(ctx, sessionID) } -func (s *redisSessionStore) GetCurActiveExport(ctx context.Context) (string, error) { - activeSessionID, err := s.client.Get(ctx, activeSessionIDKey).Result() +func (s *sessionStore) GetCurActiveExport(ctx context.Context) (string, error) { + activeSessionID, err := s.client.Get(ctx, activeSessionIDKey) if err == nil { return activeSessionID, nil } - if errors.Is(err, redis.Nil) { + if errors.Is(err, cache.ErrNotFound) { return "", ErrNoActiveExport } return "", fmt.Errorf("get: %w", err) } -func (s *redisSessionStore) CancelExport(ctx context.Context, sessionID string) error { - err := s.unlock(ctx) - if err != nil { - return err - } +func (s *sessionStore) CancelExport(ctx context.Context, sessionID string) error { + s.unlock(ctx) return s.setExportStatus(ctx, sessionID, types.ExportStatusCancel, "") } -func (s *redisSessionStore) FailExport(ctx context.Context, sessionID string, errMsg string) error { - err := s.unlock(ctx) - if err != nil { - return err - } +func (s *sessionStore) FailExport(ctx context.Context, sessionID string, errMsg string) error { + s.unlock(ctx) return s.setExportStatus(ctx, sessionID, types.ExportStatusFail, errMsg) } -func (s *redisSessionStore) FinishExport(ctx context.Context, sessionID string) error { - err := s.unlock(ctx) - if err != nil { - return err - } +func (s *sessionStore) FinishExport(ctx context.Context, sessionID string) error { + s.unlock(ctx) return s.setExportStatus(ctx, sessionID, types.ExportStatusFinish, "") } -func (s *redisSessionStore) setExportStatus( - ctx context.Context, - sessionID string, - status types.ExportStatus, - errMsg string, -) error { - result, err := s.get(ctx, sessionID) - if err != nil { - return err - } - - switch result.Status { - case types.ExportStatusCancel: - return errors.New("export already canceled") - case types.ExportStatusFail: - return errors.New("export already failed") - case types.ExportStatusFinish: - return errors.New("export already finished") - default: - result.Status = status - result.Error = errMsg - result.FinishedAt = time.Now() - return s.set(ctx, sessionID, result) - } -} - -func (s *redisSessionStore) set(ctx context.Context, sessionID string, info types.ExportInfo) error { - data, err := json.Marshal(globalToLocal(info)) - if err != nil { - return fmt.Errorf("pack export info: %w", err) - } - - err = s.client.Set(ctx, sessionID, string(data), s.exportLifetime).Err() - if err != nil { - return fmt.Errorf("set: %w", err) - } - - return nil -} - -func (s *redisSessionStore) get(ctx context.Context, sessionID string) (types.ExportInfo, error) { - str, err := s.client.Get(ctx, sessionID).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - return types.ExportInfo{}, types.ErrNotFound - } - return types.ExportInfo{}, fmt.Errorf("get: %w", err) - } - - var info exportInfo - err = json.Unmarshal([]byte(str), &info) - if err != nil { - return types.ExportInfo{}, fmt.Errorf("unpack export info: %w", err) - } - - result, err := localToGlobal(info, sessionID) - if err != nil { - return types.ExportInfo{}, fmt.Errorf("local to global: %w", err) - } - - return result, nil -} - -func (s *redisSessionStore) Lock(ctx context.Context, sessionID string) error { - ok, err := s.client.SetNX(ctx, activeSessionIDKey, sessionID, s.exportLifetime).Result() +func (s *sessionStore) Lock(ctx context.Context, sessionID string) error { + ok, err := s.client.SetNX(ctx, activeSessionIDKey, sessionID, s.lifetime) if err != nil { return fmt.Errorf("set: %w", err) } if !ok { - activeSessionID, err := s.client.Get(ctx, activeSessionIDKey).Result() + activeSessionID, err := s.client.Get(ctx, activeSessionIDKey) if err != nil { return fmt.Errorf("get active session id: %w", err) } @@ -198,17 +104,8 @@ func (s *redisSessionStore) Lock(ctx context.Context, sessionID string) error { return nil } -func (s *redisSessionStore) unlock(ctx context.Context) error { - err := s.client.Del(ctx, activeSessionIDKey).Err() - if err != nil { - return fmt.Errorf("unlock: %w", err) - } - - return nil -} - -func (s *redisSessionStore) GetAllExports(ctx context.Context) ([]types.ExportInfo, error) { - keys, err := s.getAllExports(ctx) +func (s *sessionStore) GetAllExports(ctx context.Context) ([]types.ExportInfo, error) { + keys, err := s.client.KeysByPattern(ctx, exportsKeysPattern) if err != nil { return nil, fmt.Errorf("get user exports: %w", err) } @@ -218,7 +115,7 @@ func (s *redisSessionStore) GetAllExports(ctx context.Context) ([]types.ExportIn return nil, nil } - values, err := s.client.MGet(ctx, keys...).Result() + values, err := s.client.MGet(ctx, keys...) if err != nil { return nil, fmt.Errorf("get values: %w", err) } @@ -250,33 +147,7 @@ func (s *redisSessionStore) GetAllExports(ctx context.Context) ([]types.ExportIn return result, nil } -const keysBatchSize = 10_000 - -func (s *redisSessionStore) getAllExports(ctx context.Context) ([]string, error) { - pattern := "job#*#export#*" - - var ( - cursor uint64 - result, partRes []string - err error - ) - - for { - partRes, cursor, err = s.client.Scan(ctx, cursor, pattern, keysBatchSize).Result() - if err != nil { - return nil, fmt.Errorf("redis scan: %w", err) - } - - result = append(result, partRes...) - if cursor == 0 { - break - } - } - - return result, nil -} - -func (s *redisSessionStore) ConfirmPart(ctx context.Context, sessionID string, partID int, partSize types.Size) error { +func (s *sessionStore) ConfirmPart(ctx context.Context, sessionID string, partID int, partSize types.Size) error { info, err := s.get(ctx, sessionID) if err != nil { return fmt.Errorf("get: %w", err) @@ -297,6 +168,72 @@ func (s *redisSessionStore) ConfirmPart(ctx context.Context, sessionID string, p return s.set(ctx, sessionID, info) } +func (s *sessionStore) setExportStatus( + ctx context.Context, + sessionID string, + status types.ExportStatus, + errMsg string, +) error { + result, err := s.get(ctx, sessionID) + if err != nil { + return err + } + + switch result.Status { + case types.ExportStatusCancel: + return errors.New("export already canceled") + case types.ExportStatusFail: + return errors.New("export already failed") + case types.ExportStatusFinish: + return errors.New("export already finished") + default: + result.Status = status + result.Error = errMsg + result.FinishedAt = time.Now() + return s.set(ctx, sessionID, result) + } +} + +func (s *sessionStore) set(ctx context.Context, sessionID string, info types.ExportInfo) error { + data, err := json.Marshal(globalToLocal(info)) + if err != nil { + return fmt.Errorf("pack export info: %w", err) + } + + if err := s.client.SetWithTTL(ctx, sessionID, string(data), s.lifetime); err != nil { + return fmt.Errorf("set: %w", err) + } + + return nil +} + +func (s *sessionStore) get(ctx context.Context, sessionID string) (types.ExportInfo, error) { + str, err := s.client.Get(ctx, sessionID) + if err != nil { + if errors.Is(err, cache.ErrNotFound) { + return types.ExportInfo{}, types.ErrNotFound + } + return types.ExportInfo{}, fmt.Errorf("get: %w", err) + } + + var info exportInfo + err = json.Unmarshal([]byte(str), &info) + if err != nil { + return types.ExportInfo{}, fmt.Errorf("unpack export info: %w", err) + } + + result, err := localToGlobal(info, sessionID) + if err != nil { + return types.ExportInfo{}, fmt.Errorf("local to global: %w", err) + } + + return result, nil +} + +func (s *sessionStore) unlock(ctx context.Context) { + s.client.Del(ctx, activeSessionIDKey) +} + func strToBoolArray(s string) ([]bool, error) { a := make([]bool, len(s)) for i, c := range s { From 27e1d5c1bf9e7398f3c4b37ffb4b54b4a229ad60 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Tue, 20 Oct 2026 15:30:36 +0300 Subject: [PATCH 27/29] update --- cmd/seq-ui/main.go | 4 +- internal/api/seqapi/v1/grpc/aggregation.go | 5 +- internal/api/seqapi/v1/grpc/api.go | 18 -- internal/api/seqapi/v1/grpc/cluster_status.go | 9 +- internal/api/seqapi/v1/grpc/events.go | 10 +- internal/api/seqapi/v1/grpc/events_test.go | 4 +- internal/api/seqapi/v1/grpc/fields.go | 18 +- internal/api/seqapi/v1/grpc/fields_test.go | 2 +- internal/api/seqapi/v1/grpc/histogram.go | 13 +- internal/api/seqapi/v1/grpc/limits.go | 9 +- internal/api/seqapi/v1/grpc/logs_lifespan.go | 8 +- .../api/seqapi/v1/grpc/logs_lifespan_test.go | 2 +- internal/api/seqapi/v1/grpc/search.go | 9 +- internal/api/seqapi/v1/grpc/test_data.go | 6 +- .../seqapi/v1/handler_cache/handler_cache.go | 2 +- internal/api/seqapi/v1/http/aggregation.go | 6 +- internal/api/seqapi/v1/http/aggregation_ts.go | 6 +- internal/api/seqapi/v1/http/api.go | 18 -- internal/api/seqapi/v1/http/cluster_status.go | 8 +- internal/api/seqapi/v1/http/events.go | 13 +- internal/api/seqapi/v1/http/events_test.go | 4 +- internal/api/seqapi/v1/http/export.go | 6 +- .../api/seqapi/v1/http/export_async_search.go | 6 +- internal/api/seqapi/v1/http/fields.go | 21 +- internal/api/seqapi/v1/http/fields_test.go | 2 +- internal/api/seqapi/v1/http/histogram.go | 6 +- internal/api/seqapi/v1/http/limits.go | 10 +- internal/api/seqapi/v1/http/logs_lifespan.go | 8 +- .../api/seqapi/v1/http/logs_lifespan_test.go | 2 +- internal/api/seqapi/v1/http/search.go | 13 +- .../api/seqapi/v1/http/start_async_search.go | 7 +- internal/api/seqapi/v1/http/test_data.go | 6 +- internal/app/auth/oidc.go | 11 +- internal/app/config/migrate/v1_v2.go | 20 +- internal/app/config/v2/config.go | 269 +++++++++--------- internal/app/config/v2/validate.go | 36 +-- internal/app/mw/auth.go | 4 +- internal/app/mw/env.go | 2 +- internal/app/server/init.go | 4 +- internal/pkg/cache/redis.go | 2 +- .../pkg/service/admin/permissions_test.go | 6 +- internal/pkg/service/admin/roles_test.go | 20 +- internal/pkg/service/admin/service.go | 2 +- .../pkg/service/async_searches/service.go | 8 +- internal/pkg/service/errorgroups/service.go | 8 +- internal/pkg/service/massexport/service.go | 16 +- .../service/massexport/sessionstore/redis.go | 67 +++-- 47 files changed, 300 insertions(+), 436 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index df4614c..a971d7f 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -169,7 +169,7 @@ func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { func initSeqDBClients(ctx context.Context, cfg *config.Config) (map[string]seqdb.Client, error) { clients := make(map[string]seqdb.Client, len(cfg.Clients.SeqDB)) for _, clientCfg := range cfg.Clients.SeqDB { - client, err := createSeqBDClient(ctx, &clientCfg, &cfg.Handlers.SeqAPI) + client, err := createSeqDBClient(ctx, &clientCfg, &cfg.Handlers.SeqAPI) if err != nil { return nil, fmt.Errorf("failed to create seq-db client %q: %w", clientCfg.ID, err) } @@ -179,7 +179,7 @@ func initSeqDBClients(ctx context.Context, cfg *config.Config) (map[string]seqdb return clients, nil } -func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *config.SeqAPI) (seqdb.Client, error) { +func createSeqDBClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *config.SeqAPI) (seqdb.Client, error) { var maxSearchLimit int32 if len(seqAPI.Envs) == 0 { maxSearchLimit = seqAPI.Options.Limits.MaxSearchLimit diff --git a/internal/api/seqapi/v1/grpc/aggregation.go b/internal/api/seqapi/v1/grpc/aggregation.go index 640b5ef..a935ff2 100644 --- a/internal/api/seqapi/v1/grpc/aggregation.go +++ b/internal/api/seqapi/v1/grpc/aggregation.go @@ -53,10 +53,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ } span.SetAttributes(attributes...) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } + params := a.paramsByEnv[env] if err := api_error.CheckAggregationsCount(len(req.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index d878839..3db787b 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -1,7 +1,6 @@ package grpc import ( - "fmt" "maps" "slices" "sort" @@ -135,23 +134,6 @@ func parseEnvs(cfg *config.SeqAPI) *seqapi.GetEnvsResponse { return &seqapi.GetEnvsResponse{Envs: envs} } -func (a *API) GetParams(env string) (apiParams, error) { - if len(a.config.Envs) == 0 { - return a.paramsByEnv[""], nil - } - - if env == "" { - env = a.config.DefaultEnv - } - - params, exists := a.paramsByEnv[env] - if !exists { - return apiParams{}, fmt.Errorf("env '%s' not found", env) - } - - return params, nil -} - func checkLimitOffset(limit, offset int32) error { if limit < 0 { return types.NewErrInvalidRequestField("'limit' must be non-negative") diff --git a/internal/api/seqapi/v1/grpc/cluster_status.go b/internal/api/seqapi/v1/grpc/cluster_status.go index 06f6b0e..8e068ca 100644 --- a/internal/api/seqapi/v1/grpc/cluster_status.go +++ b/internal/api/seqapi/v1/grpc/cluster_status.go @@ -3,9 +3,6 @@ package grpc import ( "context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" @@ -15,11 +12,7 @@ func (a *API) Status(ctx context.Context, req *seqapi.StatusRequest) (*seqapi.St ctx, span := tracing.StartSpan(ctx, "seqapi_v1_status") defer span.End() - env := types.GetEnvKey(ctx) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } + params := a.paramsByEnv[types.GetEnvKey(ctx)] return params.client.Status(ctx, req) } diff --git a/internal/api/seqapi/v1/grpc/events.go b/internal/api/seqapi/v1/grpc/events.go index ca892c4..d122fd0 100644 --- a/internal/api/seqapi/v1/grpc/events.go +++ b/internal/api/seqapi/v1/grpc/events.go @@ -5,8 +5,6 @@ import ( "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/app/types" @@ -32,14 +30,12 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap } span.SetAttributes(attributes...) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } + params := a.paramsByEnv[env] if cached, ok := params.caches.Events.Get(ctx, req.Id); ok { event := &seqapi.Event{} - if err = proto.Unmarshal([]byte(cached), event); err == nil { + err := proto.Unmarshal([]byte(cached), event) + if err == nil { if params.masker != nil { params.masker.Mask(event.Data) } diff --git a/internal/api/seqapi/v1/grpc/events_test.go b/internal/api/seqapi/v1/grpc/events_test.go index cf84867..8bb9956 100644 --- a/internal/api/seqapi/v1/grpc/events_test.go +++ b/internal/api/seqapi/v1/grpc/events_test.go @@ -106,7 +106,7 @@ func TestGetEvent(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Events: &config.HandlerCache{TTL: cacheTTL}, + Events: &config.HandlerCache{TTL: new(cacheTTL)}, }, }, }, @@ -373,7 +373,7 @@ func TestGetEventWithMasking(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Events: &config.HandlerCache{TTL: cacheTTL}, + Events: &config.HandlerCache{TTL: new(cacheTTL)}, }, Masking: tt.maskingCfg, }, diff --git a/internal/api/seqapi/v1/grpc/fields.go b/internal/api/seqapi/v1/grpc/fields.go index 7bc535c..4edd742 100644 --- a/internal/api/seqapi/v1/grpc/fields.go +++ b/internal/api/seqapi/v1/grpc/fields.go @@ -4,8 +4,6 @@ import ( "context" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/app/types" @@ -18,15 +16,12 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_fields") defer span.End() - env := types.GetEnvKey(ctx) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } + params := a.paramsByEnv[types.GetEnvKey(ctx)] if cached, ok := params.caches.Fields.Get(ctx, ""); ok { var resp seqapi.GetFieldsResponse - if err = proto.Unmarshal([]byte(cached), &resp); err == nil { + err := proto.Unmarshal([]byte(cached), &resp) + if err == nil { return &seqapi.GetFieldsResponse{ Fields: resp.GetFields(), SystemFields: params.systemFields, @@ -55,11 +50,8 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq } func (a *API) GetPinnedFields(ctx context.Context, _ *seqapi.GetFieldsRequest) (*seqapi.GetFieldsResponse, error) { - env := types.GetEnvKey(ctx) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } + params := a.paramsByEnv[types.GetEnvKey(ctx)] + return &seqapi.GetFieldsResponse{ Fields: params.pinnedFields, }, nil diff --git a/internal/api/seqapi/v1/grpc/fields_test.go b/internal/api/seqapi/v1/grpc/fields_test.go index 0cdc9a6..9effbda 100644 --- a/internal/api/seqapi/v1/grpc/fields_test.go +++ b/internal/api/seqapi/v1/grpc/fields_test.go @@ -186,7 +186,7 @@ func TestGetFieldsCached(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Fields: &config.HandlerCache{TTL: ttl}, + Fields: &config.HandlerCache{TTL: new(ttl)}, }, }, }, diff --git a/internal/api/seqapi/v1/grpc/histogram.go b/internal/api/seqapi/v1/grpc/histogram.go index 261fbd4..23304f7 100644 --- a/internal/api/seqapi/v1/grpc/histogram.go +++ b/internal/api/seqapi/v1/grpc/histogram.go @@ -4,8 +4,6 @@ import ( "context" "go.opentelemetry.io/otel/attribute" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -45,14 +43,7 @@ func (a *API) GetHistogram(ctx context.Context, req *seqapi.GetHistogramRequest) } span.SetAttributes(attributes...) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } + params := a.paramsByEnv[env] - resp, err := params.client.GetHistogram(ctx, req) - if err != nil { - return nil, err - } - return resp, nil + return params.client.GetHistogram(ctx, req) } diff --git a/internal/api/seqapi/v1/grpc/limits.go b/internal/api/seqapi/v1/grpc/limits.go index fb2a66d..09264f7 100644 --- a/internal/api/seqapi/v1/grpc/limits.go +++ b/internal/api/seqapi/v1/grpc/limits.go @@ -3,19 +3,12 @@ package grpc import ( "context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) func (a *API) GetLimits(ctx context.Context, _ *seqapi.GetLimitsRequest) (*seqapi.GetLimitsResponse, error) { - env := types.GetEnvKey(ctx) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } + params := a.paramsByEnv[types.GetEnvKey(ctx)] return &seqapi.GetLimitsResponse{ MaxSearchLimit: params.options.Limits.MaxSearchLimit, diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan.go b/internal/api/seqapi/v1/grpc/logs_lifespan.go index d8a17d2..63aa9c1 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan.go @@ -8,8 +8,6 @@ import ( "time" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/durationpb" "github.com/ozontech/seq-ui/internal/api/grpcutil" @@ -24,11 +22,7 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_logs_lifespan") defer span.End() - env := types.GetEnvKey(ctx) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } + params := a.paramsByEnv[types.GetEnvKey(ctx)] if countStr, ok := params.caches.LogsLifespan.Get(ctx, ""); ok { count, err := strconv.Atoi(countStr) diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index e71d99f..c93dfb9 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -108,7 +108,7 @@ func TestGetLogsLifespan(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - LogsLifespan: &config.HandlerCache{TTL: cacheTTL}, + LogsLifespan: &config.HandlerCache{TTL: new(cacheTTL)}, }, }, }, diff --git a/internal/api/seqapi/v1/grpc/search.go b/internal/api/seqapi/v1/grpc/search.go index 6ca4649..82c2b31 100644 --- a/internal/api/seqapi/v1/grpc/search.go +++ b/internal/api/seqapi/v1/grpc/search.go @@ -62,12 +62,6 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se if env != "" { attributes = append(attributes, attribute.String("env", env)) } - - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - if req.Histogram != nil && req.Histogram.Interval != "" { attributes = append(attributes, attribute.KeyValue{ Key: "histogram_interval", @@ -81,9 +75,10 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se Value: attribute.StringValue(string(aggregations)), }) } - span.SetAttributes(attributes...) + params := a.paramsByEnv[env] + if err := api_error.CheckSearchLimit(req.Limit, params.options.Limits.MaxSearchLimit); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index a7caef0..963a77a 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -41,13 +41,13 @@ func setupTestAPI(data test.APITestData) *API { caches := data.Cfg.Options.Caches for env, params := range api.paramsByEnv { if caches.Events != nil { - params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: caches.Events.TTL} + params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: *caches.Events.TTL} } if caches.Fields != nil { - params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: caches.Fields.TTL} + params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: *caches.Fields.TTL} } if caches.LogsLifespan != nil { - params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: caches.LogsLifespan.TTL} + params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: *caches.LogsLifespan.TTL} } api.paramsByEnv[env] = params } diff --git a/internal/api/seqapi/v1/handler_cache/handler_cache.go b/internal/api/seqapi/v1/handler_cache/handler_cache.go index 6c0ec1a..b1a26a7 100644 --- a/internal/api/seqapi/v1/handler_cache/handler_cache.go +++ b/internal/api/seqapi/v1/handler_cache/handler_cache.go @@ -47,7 +47,7 @@ func newCache(cfg *config.HandlerCache, keyPrefix string) *Cache { return nil } - return &Cache{Cache: c, KeyPrefix: keyPrefix, TTL: cfg.TTL} + return &Cache{Cache: c, KeyPrefix: keyPrefix, TTL: *cfg.TTL} } func (c *Cache) Get(ctx context.Context, key string) (string, bool) { diff --git a/internal/api/seqapi/v1/http/aggregation.go b/internal/api/seqapi/v1/http/aggregation.go index 3195a82..baf667f 100644 --- a/internal/api/seqapi/v1/http/aggregation.go +++ b/internal/api/seqapi/v1/http/aggregation.go @@ -74,11 +74,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(attributes...) - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[env] if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { wr.Error(err, http.StatusBadRequest) diff --git a/internal/api/seqapi/v1/http/aggregation_ts.go b/internal/api/seqapi/v1/http/aggregation_ts.go index 9fa4206..0650078 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts.go +++ b/internal/api/seqapi/v1/http/aggregation_ts.go @@ -68,11 +68,7 @@ func (a *API) serveGetAggregationTs(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(attributes...) - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[env] if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { wr.Error(err, http.StatusBadRequest) diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index 84a30a7..50db910 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -1,7 +1,6 @@ package http import ( - "fmt" "maps" "slices" "sort" @@ -181,23 +180,6 @@ const ( aecTooManyFractionsHit apiErrorCode = "ERROR_CODE_TOO_MANY_FRACTIONS_HIT" ) -func (a *API) GetEnvParams(env string) (apiParams, error) { - if len(a.config.Envs) == 0 { - return a.paramsByEnv[""], nil - } - - if env == "" { - env = a.config.DefaultEnv - } - - params, exists := a.paramsByEnv[env] - if !exists { - return apiParams{}, fmt.Errorf("env '%s' not found", env) - } - - return params, nil -} - func apiErrorCodeFromProto(proto seqapi.ErrorCode) apiErrorCode { switch proto { case seqapi.ErrorCode_ERROR_CODE_UNSPECIFIED: diff --git a/internal/api/seqapi/v1/http/cluster_status.go b/internal/api/seqapi/v1/http/cluster_status.go index 4f1ca1b..411c2d2 100644 --- a/internal/api/seqapi/v1/http/cluster_status.go +++ b/internal/api/seqapi/v1/http/cluster_status.go @@ -25,13 +25,7 @@ func (a *API) serveStatus(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - - env := types.GetEnvKey(ctx) - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[types.GetEnvKey(ctx)] resp, err := params.client.Status(ctx, &seqapi.StatusRequest{}) if err != nil { diff --git a/internal/api/seqapi/v1/http/events.go b/internal/api/seqapi/v1/http/events.go index a1dcc0b..6d306ad 100644 --- a/internal/api/seqapi/v1/http/events.go +++ b/internal/api/seqapi/v1/http/events.go @@ -45,15 +45,12 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(attributes...) - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[env] if cached, ok := params.caches.Events.Get(ctx, id); ok { e := &seqapi.Event{} - if err = proto.Unmarshal([]byte(cached), e); err == nil { + err := proto.Unmarshal([]byte(cached), e) + if err == nil { if params.masker != nil { params.masker.Mask(e.Data) } @@ -81,9 +78,7 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { } } - eventResp := getEventResponseFromProto(resp) - - wr.WriteJson(eventResp) + wr.WriteJson(getEventResponseFromProto(resp)) } type event struct { diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index 7682771..b679f88 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -122,7 +122,7 @@ func TestServeGetEvent(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Events: &config.HandlerCache{TTL: cacheTTL}, + Events: &config.HandlerCache{TTL: new(cacheTTL)}, }, }, }, @@ -384,7 +384,7 @@ func TestGetEventWithMasking(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Events: &config.HandlerCache{TTL: cacheTTL}, + Events: &config.HandlerCache{TTL: new(cacheTTL)}, }, Masking: tt.maskingCfg, }, diff --git a/internal/api/seqapi/v1/http/export.go b/internal/api/seqapi/v1/http/export.go index 7cee32a..6a0f1eb 100644 --- a/internal/api/seqapi/v1/http/export.go +++ b/internal/api/seqapi/v1/http/export.go @@ -88,11 +88,7 @@ func (a *API) serveExport(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(attributes...) - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[env] if params.exportLimiter.Limited(userStr) { metric.ServerExportRequestLimits.Inc() diff --git a/internal/api/seqapi/v1/http/export_async_search.go b/internal/api/seqapi/v1/http/export_async_search.go index 26e0a15..4ee39f9 100644 --- a/internal/api/seqapi/v1/http/export_async_search.go +++ b/internal/api/seqapi/v1/http/export_async_search.go @@ -91,11 +91,7 @@ func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { return } - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[env] if params.exportLimiter.Limited(userStr) { metric.ServerExportRequestLimits.Inc() diff --git a/internal/api/seqapi/v1/http/fields.go b/internal/api/seqapi/v1/http/fields.go index d5a5fb2..deeeb08 100644 --- a/internal/api/seqapi/v1/http/fields.go +++ b/internal/api/seqapi/v1/http/fields.go @@ -27,17 +27,12 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - - env := types.GetEnvKey(ctx) - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[types.GetEnvKey(ctx)] if cached, ok := params.caches.Fields.Get(ctx, ""); ok { var resp seqapi.GetFieldsResponse - if err = proto.Unmarshal([]byte(cached), &resp); err == nil { + err := proto.Unmarshal([]byte(cached), &resp) + if err == nil { wr.WriteJson(getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), SystemFields: params.systemFields, @@ -82,15 +77,9 @@ func (a *API) serveGetPinnedFields(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) + params := a.paramsByEnv[types.GetEnvKey(ctx)] - env := types.GetEnvKey(ctx) - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } - - httputil.NewWriter(w).WriteJson(getFieldsResponse{ + wr.WriteJson(getFieldsResponse{ Fields: params.pinnedFields, }) } diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index a8c5de4..274c55e 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -97,7 +97,7 @@ func TestServeGetFields(t *testing.T) { cfg: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ Fields: &config.HandlerCache{ - TTL: time.Hour, + TTL: new(time.Hour), }, }, }, diff --git a/internal/api/seqapi/v1/http/histogram.go b/internal/api/seqapi/v1/http/histogram.go index 8db4d20..7f77366 100644 --- a/internal/api/seqapi/v1/http/histogram.go +++ b/internal/api/seqapi/v1/http/histogram.go @@ -67,11 +67,7 @@ func (a *API) serveGetHistogram(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(attributes...) - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[env] resp, err := params.client.GetHistogram(ctx, httpReq.toProto()) if err != nil { diff --git a/internal/api/seqapi/v1/http/limits.go b/internal/api/seqapi/v1/http/limits.go index 6d04d8b..582d716 100644 --- a/internal/api/seqapi/v1/http/limits.go +++ b/internal/api/seqapi/v1/http/limits.go @@ -21,15 +21,9 @@ func (a *API) serveGetLimits(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - env := types.GetEnvKey(ctx) + params := a.paramsByEnv[types.GetEnvKey(ctx)] - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } - - httputil.NewWriter(w).WriteJson(getLimitsResponse{ + wr.WriteJson(getLimitsResponse{ MaxSearchLimit: int(params.options.Limits.MaxSearchLimit), MaxExportLimit: int(params.options.Limits.MaxExportLimit), MaxParallelExportRequests: params.options.Limits.MaxParallelExportRequests, diff --git a/internal/api/seqapi/v1/http/logs_lifespan.go b/internal/api/seqapi/v1/http/logs_lifespan.go index ebf38e5..4db7b24 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan.go +++ b/internal/api/seqapi/v1/http/logs_lifespan.go @@ -29,13 +29,7 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - - env := types.GetEnvKey(ctx) - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[types.GetEnvKey(ctx)] if resStr, ok := params.caches.LogsLifespan.Get(ctx, ""); ok { res, err := strconv.Atoi(resStr) diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index 616cce6..fa018de 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -106,7 +106,7 @@ func TestServeGetLogsLifespan(t *testing.T) { Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ LogsLifespan: &config.HandlerCache{ - TTL: cacheTTL, + TTL: new(cacheTTL), }, }, }, diff --git a/internal/api/seqapi/v1/http/search.go b/internal/api/seqapi/v1/http/search.go index 3285fd3..47a9f51 100644 --- a/internal/api/seqapi/v1/http/search.go +++ b/internal/api/seqapi/v1/http/search.go @@ -86,13 +86,6 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { if env != "" { attributes = append(attributes, attribute.String("env", env)) } - - params, err := a.GetEnvParams(env) - if err != nil { - wr.Error(err, http.StatusBadRequest) - return - } - if httpReq.Histogram.Interval != "" { attributes = append(attributes, attribute.KeyValue{ Key: "histogram_interval", @@ -108,6 +101,8 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(attributes...) + params := a.paramsByEnv[env] + if err := api_error.CheckSearchLimit(httpReq.Limit, params.options.Limits.MaxSearchLimit); err != nil { wr.Error(err, http.StatusBadRequest) return @@ -168,8 +163,8 @@ type searchRequest struct { Query string `json:"query"` From time.Time `json:"from" format:"date-time"` To time.Time `json:"to" format:"date-time"` - Limit int32 `json:"limit"` - Offset int32 `json:"offset"` + Limit int32 `json:"limit" format:"int32"` + Offset int32 `json:"offset" format:"int32"` WithTotal bool `json:"withTotal"` Aggregations aggregationQueries `json:"aggregations"` Histogram struct { diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index d70ac92..1480e32 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -40,12 +40,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { env := types.GetEnvKey(ctx) svc := a.asyncSearchesByEnv[env] - - params, err := a.GetEnvParams(env) - if err != nil { - httputil.NewWriter(w).Error(err, http.StatusBadRequest) - return - } + params := a.paramsByEnv[env] var httpReq startAsyncSearchRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 17d5f5d..5661077 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -46,13 +46,13 @@ func setupTestAPI(data test.APITestData) *API { caches := data.Cfg.Options.Caches for env, params := range api.paramsByEnv { if caches.Events != nil { - params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: caches.Events.TTL} + params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: *caches.Events.TTL} } if caches.Fields != nil { - params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: caches.Fields.TTL} + params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: *caches.Fields.TTL} } if caches.LogsLifespan != nil { - params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: caches.LogsLifespan.TTL} + params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: *caches.LogsLifespan.TTL} } api.paramsByEnv[env] = params } diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 783b19f..f26fca9 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -38,11 +38,12 @@ type oidcProvider struct { cache cache.Cache cacheSecretKey string + cacheKeyPrefix string allowedClients []string } -func NewOIDCProvider(ctx context.Context, cfg *config.OIDC) (OIDCProvider, error) { +func NewOIDCProvider(ctx context.Context, cfg *config.OIDC, env string) (OIDCProvider, error) { var ( err error oidcCache cache.Cache @@ -88,10 +89,16 @@ func NewOIDCProvider(ctx context.Context, cfg *config.OIDC) (OIDCProvider, error return nil, fmt.Errorf("no valid OIDC auth urls") } + cacheKeyPrefix := oidcCacheKeyPrefix + if env != "" { + cacheKeyPrefix += "_" + env + } + return &oidcProvider{ verifiers: verifiers, cache: oidcCache, cacheSecretKey: cfg.CacheSecretKey, + cacheKeyPrefix: cacheKeyPrefix, allowedClients: cfg.AllowedClients, }, nil } @@ -108,7 +115,7 @@ func (p *oidcProvider) Verify(ctx context.Context, token string) (OIDCToken, err ) if p.cacheSecretKey != "" { - cacheKey = fmt.Sprintf("%s_%s", oidcCacheKeyPrefix, hashToken(token, p.cacheSecretKey)) + cacheKey = fmt.Sprintf("%s_%s", p.cacheKeyPrefix, hashToken(token, p.cacheSecretKey)) if userName, err := p.cache.Get(ctx, cacheKey); err == nil { return OIDCToken{UserName: userName}, nil } diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index 213527e..a8de4fb 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -275,7 +275,10 @@ func (m *v1tov2) migrateCache(src v1.Cache, sessionStore *v1.SessionStore) v2.Ca } if sessionStore != nil { - cache.Redis = append(cache.Redis, m.migrateRedis(&sessionStore.Redis, v2.DefaultMassExportRedisID, "")) + // in v1 session store ignored key_prefix + redis := m.migrateRedis(&sessionStore.Redis, v2.DefaultMassExportRedisID, "") + redis.KeyPrefix = "" + cache.Redis = append(cache.Redis, redis) } return cache @@ -354,11 +357,11 @@ func (m *v1tov2) migrateSeqAPIOptions(src *v1.SeqAPIOptions, v1Cache v1.Cache) v Caches: v2.SeqAPICaches{ Events: &v2.HandlerCache{ ID: redisWithInmemID, - TTL: src.EventsCacheTTL, + TTL: m.migrateTTL(src.EventsCacheTTL), }, LogsLifespan: &v2.HandlerCache{ ID: redisOrInmemID, - TTL: src.LogsLifespanCacheTTL, + TTL: m.migrateTTL(src.LogsLifespanCacheTTL), }, }, } @@ -367,7 +370,7 @@ func (m *v1tov2) migrateSeqAPIOptions(src *v1.SeqAPIOptions, v1Cache v1.Cache) v if src.FieldsCacheTTL > 0 { dst.Caches.Fields = &v2.HandlerCache{ ID: v2.DefaultInmemID, - TTL: src.FieldsCacheTTL, + TTL: m.migrateTTL(src.FieldsCacheTTL), } } @@ -551,13 +554,20 @@ func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { if v1Cache.Redis != nil { dst.Options.Cache = &v2.HandlerCache{ ID: v2.DefaultRedisWithoutInmemID, - TTL: src.CacheTTL, + TTL: m.migrateTTL(src.CacheTTL), } } return dst } +func (m *v1tov2) migrateTTL(ttl time.Duration) *time.Duration { + if ttl <= 0 { + return nil + } + return &ttl +} + // defaultCacheIDs returns cache ids same as in v1. // inmemWithRedisID - inmemory+redis or inmemory (v1 events, oidc) // redisOrInmemID - redis or inmemory (v1 logslifespan) diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 95a7d27..96c4b85 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -237,34 +237,34 @@ type Config struct { Server Server `yaml:"server"` Clients Clients `yaml:"clients"` Handlers Handlers `yaml:"handlers"` - Cache Cache `yaml:"cache"` - DB *DB `yaml:"db"` + Cache Cache `yaml:"cache,omitempty"` + DB *DB `yaml:"db,omitempty"` } type CORS struct { - AllowedOrigins []string `yaml:"allowed_origins"` - AllowedMethods []string `yaml:"allowed_methods"` - AllowedHeaders []string `yaml:"allowed_headers"` - ExposedHeaders []string `yaml:"exposed_headers"` - MaxAge int `yaml:"max_age"` - AllowCredentials bool `yaml:"allow_credentials"` - OptionsPassthrough bool `yaml:"options_passthrough"` + AllowedOrigins []string `yaml:"allowed_origins,omitempty"` + AllowedMethods []string `yaml:"allowed_methods,omitempty"` + AllowedHeaders []string `yaml:"allowed_headers,omitempty"` + ExposedHeaders []string `yaml:"exposed_headers,omitempty"` + MaxAge int `yaml:"max_age,omitempty"` + AllowCredentials bool `yaml:"allow_credentials,omitempty"` + OptionsPassthrough bool `yaml:"options_passthrough,omitempty"` } type TLS struct { - CACert string `yaml:"ca_cert"` - ClientCert string `yaml:"client_cert"` - ClientKey string `yaml:"client_key"` - Insecure bool `yaml:"insecure"` + CACert string `yaml:"ca_cert,omitempty"` + ClientCert string `yaml:"client_cert,omitempty"` + ClientKey string `yaml:"client_key,omitempty"` + Insecure bool `yaml:"insecure,omitempty"` } type OIDC struct { - CacheID string `yaml:"cache_id"` - CacheSecretKey string `yaml:"cache_secret_key"` - SkipVerify bool `yaml:"skip_verify"` + CacheID string `yaml:"cache_id,omitempty"` + CacheSecretKey string `yaml:"cache_secret_key,omitempty"` + SkipVerify bool `yaml:"skip_verify,omitempty"` AuthURLs []string `yaml:"auth_urls"` - AllowedClients []string `yaml:"allowed_clients"` - TLS *TLS `yaml:"tls"` + AllowedClients []string `yaml:"allowed_clients,omitempty"` + TLS *TLS `yaml:"tls,omitempty"` } type DB struct { @@ -275,7 +275,7 @@ type DB struct { Pass string `yaml:"pass"` RequestTimeout time.Duration `yaml:"request_timeout"` ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` - UsePreparedStatements *bool `yaml:"use_prepared_statements"` + UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` } func (db *DB) ConnString() string { @@ -285,14 +285,14 @@ func (db *DB) ConnString() string { type ( RateLimiter struct { RatePerSec int `yaml:"rate_per_sec"` - MaxBurst int `yaml:"max_burst"` - StoreMaxKeys int `yaml:"store_max_keys"` - PerHandler bool `yaml:"per_handler"` + MaxBurst int `yaml:"max_burst,omitempty"` + StoreMaxKeys int `yaml:"store_max_keys,omitempty"` + PerHandler bool `yaml:"per_handler,omitempty"` } RateLimiters struct { Default RateLimiter `yaml:"default"` - SpecialUsers map[string]RateLimiter `yaml:"spec_users"` + SpecialUsers map[string]RateLimiter `yaml:"spec_users,omitempty"` } ApiToRateLimiters map[string]RateLimiters @@ -300,27 +300,27 @@ type ( type Inmemory struct { ID string `yaml:"id"` - NumCounters int64 `yaml:"num_counters"` - MaxCost int64 `yaml:"max_cost"` - BufferItems int64 `yaml:"buffer_items"` + NumCounters int64 `yaml:"num_counters,omitempty"` + MaxCost int64 `yaml:"max_cost,omitempty"` + BufferItems int64 `yaml:"buffer_items,omitempty"` } type Redis struct { ID string `yaml:"id"` - WithInmemID string `yaml:"with_inmem_id"` - KeyPrefix string `yaml:"key_prefix"` + WithInmemID string `yaml:"with_inmem_id,omitempty"` + KeyPrefix string `yaml:"key_prefix,omitempty"` Addr string `yaml:"addr"` - Username string `yaml:"username"` - Password string `yaml:"password"` - MaxRetries int `yaml:"max_retries"` - Timeout time.Duration `yaml:"timeout"` - MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + Username string `yaml:"username,omitempty"` + Password string `yaml:"password,omitempty"` + MaxRetries int `yaml:"max_retries,omitempty"` + Timeout time.Duration `yaml:"timeout,omitempty"` + MinRetryBackoff time.Duration `yaml:"min_retry_backoff,omitempty"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff,omitempty"` } type Cache struct { - Inmemory []Inmemory `yaml:"inmemory"` - Redis []Redis `yaml:"redis"` + Inmemory []Inmemory `yaml:"inmemory,omitempty"` + Redis []Redis `yaml:"redis,omitempty"` } func (c *Cache) InmemByID(id string) *Inmemory { @@ -350,18 +350,18 @@ type S3 struct { AccessKeyID string `yaml:"access_key_id"` SecretAccessKey string `yaml:"secret_access_key"` BucketName string `yaml:"bucket_name"` - EnableSSL bool `yaml:"enable_ssl"` + EnableSSL bool `yaml:"enable_ssl,omitempty"` } type DownloadParams struct { - BatchSize uint64 `yaml:"batch_size"` - Delay time.Duration `yaml:"delay"` - DelayBackoff Backoff `yaml:"delay_backoff"` + BatchSize uint64 `yaml:"batch_size,omitempty"` + Delay time.Duration `yaml:"delay,omitempty"` + DelayBackoff Backoff `yaml:"delay_backoff,omitempty"` } type SessionStore struct { - Lifetime time.Duration `yaml:"lifetime"` CacheID string `yaml:"cache_id"` + Lifetime time.Duration `yaml:"lifetime,omitempty"` } type FileStore struct { @@ -370,45 +370,45 @@ type FileStore struct { type MassExport struct { Options MassExportOptions `yaml:"options"` - Envs map[string]MassExportEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` + Envs map[string]MassExportEnv `yaml:"envs,omitempty"` + DefaultEnv string `yaml:"default_env,omitempty"` } type MassExportEnv struct { SeqDBID string `yaml:"seq_db_id"` - Options *MassExportOptions `yaml:"options"` + Options *MassExportOptions `yaml:"options,omitempty"` } type MassExportOptions struct { WorkersCount int `yaml:"workers_count"` - TasksChannelSize int `yaml:"tasks_channel_size"` - PartLength time.Duration `yaml:"part_length"` + TasksChannelSize int `yaml:"tasks_channel_size,omitempty"` + PartLength time.Duration `yaml:"part_length,omitempty"` URLPrefix string `yaml:"url_prefix"` - AllowedUsers []string `yaml:"allowed_users"` + AllowedUsers []string `yaml:"allowed_users,omitempty"` FileStore FileStore `yaml:"file_store"` SessionStore SessionStore `yaml:"session_store"` - DownloadParams DownloadParams `yaml:"download_params"` + DownloadParams DownloadParams `yaml:"download_params,omitempty"` } type HTTP struct { Addr string `yaml:"addr"` - ReadHeaderTimeout time.Duration `yaml:"read_header_timeout"` - ReadTimeout time.Duration `yaml:"read_timeout"` - WriteTimeout time.Duration `yaml:"write_timeout"` - CORS *CORS `yaml:"cors"` + ReadHeaderTimeout time.Duration `yaml:"read_header_timeout,omitempty"` + ReadTimeout time.Duration `yaml:"read_timeout,omitempty"` + WriteTimeout time.Duration `yaml:"write_timeout,omitempty"` + CORS *CORS `yaml:"cors,omitempty"` } type GRPC struct { Addr string `yaml:"addr"` - ConnectionTimeout time.Duration `yaml:"connection_timeout"` + ConnectionTimeout time.Duration `yaml:"connection_timeout,omitempty"` } type Debug struct { Addr string `yaml:"addr"` - ReadHeaderTimeout time.Duration `yaml:"read_header_timeout"` - ReadTimeout time.Duration `yaml:"read_timeout"` - WriteTimeout time.Duration `yaml:"write_timeout"` - CORS *CORS `yaml:"cors"` + ReadHeaderTimeout time.Duration `yaml:"read_header_timeout,omitempty"` + ReadTimeout time.Duration `yaml:"read_timeout,omitempty"` + WriteTimeout time.Duration `yaml:"write_timeout,omitempty"` + CORS *CORS `yaml:"cors,omitempty"` } type JWT struct { @@ -417,76 +417,77 @@ type JWT struct { type Auth struct { Options AuthOptions `yaml:"options"` - Envs map[string]AuthOptions `yaml:"envs"` + Envs map[string]AuthOptions `yaml:"envs,omitempty"` } type AuthOptions struct { - OIDC *OIDC `yaml:"oidc"` - JWT *JWT `yaml:"jwt"` + OIDC *OIDC `yaml:"oidc,omitempty"` + JWT *JWT `yaml:"jwt,omitempty"` } type Server struct { - HTTP *HTTP `yaml:"http"` - GRPC *GRPC `yaml:"grpc"` + HTTP *HTTP `yaml:"http,omitempty"` + GRPC *GRPC `yaml:"grpc,omitempty"` Debug Debug `yaml:"debug"` - Auth *Auth `yaml:"auth"` - RateLimiting *RateLimiting `yaml:"rate_limiting"` + Auth *Auth `yaml:"auth,omitempty"` + RateLimiting *RateLimiting `yaml:"rate_limiting,omitempty"` } type RateLimiting struct { - Options ApiToRateLimiters `yaml:"options"` - Envs map[string]ApiToRateLimiters `yaml:"envs"` + // rl options are optional, without them requests not covered by envs are not limited. + Options ApiToRateLimiters `yaml:"options,omitempty"` + Envs map[string]ApiToRateLimiters `yaml:"envs,omitempty"` } type KeepaliveParams struct { // After a duration of this time if the client doesn't see any activity it // pings the server to see if the transport is still alive. // If set below 10s, a minimum value of 10s will be used instead. - Time time.Duration `yaml:"time"` + Time time.Duration `yaml:"time,omitempty"` // After having pinged for keepalive check, the client waits for a duration // of Timeout and if no activity is seen even after that the connection is // closed. If set below 1s, a minimum value of 1s will be used instead. - Timeout time.Duration `yaml:"timeout"` + Timeout time.Duration `yaml:"timeout,omitempty"` // If true, client sends keepalive pings even with no active RPCs. If false, // when there are no active RPCs, Time and Timeout will be ignored and no // keepalive pings will be sent. False by default. - PermitWithoutStream bool `yaml:"permit_without_stream"` + PermitWithoutStream bool `yaml:"permit_without_stream,omitempty"` } type GRPCParams struct { - Keepalive *KeepaliveParams `yaml:"keepalive"` + Keepalive *KeepaliveParams `yaml:"keepalive,omitempty"` } type SeqDBClient struct { ID string `yaml:"id"` Timeout time.Duration `yaml:"timeout"` - AvgDocSize int `yaml:"avg_doc_size"` + AvgDocSize int `yaml:"avg_doc_size,omitempty"` Addrs []string `yaml:"addrs"` - RequestRetries int `yaml:"request_retries"` - RequestRetriesBackoff *Backoff `yaml:"request_retries_backoff"` - ClientMode string `yaml:"client_mode"` - GRPCParams *GRPCParams `yaml:"grpc_params"` + RequestRetries int `yaml:"request_retries,omitempty"` + RequestRetriesBackoff *Backoff `yaml:"request_retries_backoff,omitempty"` + ClientMode string `yaml:"client_mode,omitempty"` + GRPCParams *GRPCParams `yaml:"grpc_params,omitempty"` } type Backoff struct { - InitialInterval time.Duration `yaml:"initial_interval"` - MaxInterval time.Duration `yaml:"max_interval"` + InitialInterval time.Duration `yaml:"initial_interval,omitempty"` + MaxInterval time.Duration `yaml:"max_interval,omitempty"` } type CHClient struct { ID string `yaml:"id"` Addrs []string `yaml:"addrs"` - Database string `yaml:"database"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Sharded bool `yaml:"sharded"` - DialTimeout time.Duration `yaml:"dial_timeout"` - ReadTimeout time.Duration `yaml:"read_timeout"` + Database string `yaml:"database,omitempty"` + Username string `yaml:"username,omitempty"` + Password string `yaml:"password,omitempty"` + Sharded bool `yaml:"sharded,omitempty"` + DialTimeout time.Duration `yaml:"dial_timeout,omitempty"` + ReadTimeout time.Duration `yaml:"read_timeout,omitempty"` } type Clients struct { SeqDB []SeqDBClient `yaml:"seq_db"` - ClickHouse []CHClient `yaml:"clickhouse"` + ClickHouse []CHClient `yaml:"clickhouse,omitempty"` } func (c *Clients) ClickHouseByID(id string) *CHClient { @@ -502,25 +503,25 @@ func (c *Clients) ClickHouseByID(id string) *CHClient { type Handlers struct { SeqAPI SeqAPI `yaml:"seq_api"` - MassExport *MassExport `yaml:"mass_export"` - AsyncSearch *AsyncSearch `yaml:"async_search"` - ErrorGroups *ErrorGroups `yaml:"error_groups"` - Admin *Admin `yaml:"admin"` + MassExport *MassExport `yaml:"mass_export,omitempty"` + AsyncSearch *AsyncSearch `yaml:"async_search,omitempty"` + ErrorGroups *ErrorGroups `yaml:"error_groups,omitempty"` + Admin *Admin `yaml:"admin,omitempty"` } type Admin struct { Options AdminOptions `yaml:"options"` - Envs map[string]AdminEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` + Envs map[string]AdminEnv `yaml:"envs,omitempty"` + DefaultEnv string `yaml:"default_env,omitempty"` } type AdminEnv struct { - Options *AdminOptions `yaml:"options"` + Options *AdminOptions `yaml:"options,omitempty"` } type AdminOptions struct { - SuperUsers []string `yaml:"super_users"` - Cache *HandlerCache `yaml:"cache"` + SuperUsers []string `yaml:"super_users,omitempty"` + Cache *HandlerCache `yaml:"cache,omitempty"` } type Field struct { @@ -530,61 +531,61 @@ type Field struct { type SeqAPI struct { Options SeqAPIOptions `yaml:"options"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` + Envs map[string]SeqAPIEnv `yaml:"envs,omitempty"` + DefaultEnv string `yaml:"default_env,omitempty"` } type SeqAPIEnv struct { SeqDBID string `yaml:"seq_db_id"` - Options *SeqAPIOptions `yaml:"options"` + Options *SeqAPIOptions `yaml:"options,omitempty"` } type SeqAPIOptions struct { - Limits SeqAPILimits `yaml:"limits"` - Caches SeqAPICaches `yaml:"caches"` - Masking *Masking `yaml:"masking"` - PinnedFields []Field `yaml:"pinned_fields"` - SystemFields []Field `yaml:"system_fields"` + Limits SeqAPILimits `yaml:"limits,omitempty"` + Caches SeqAPICaches `yaml:"caches,omitempty"` + Masking *Masking `yaml:"masking,omitempty"` + PinnedFields []Field `yaml:"pinned_fields,omitempty"` + SystemFields []Field `yaml:"system_fields,omitempty"` } type SeqAPILimits struct { - MaxSearchLimit int32 `yaml:"max_search_limit"` - MaxSearchTotal int64 `yaml:"max_search_total"` - MaxSearchOffset int32 `yaml:"max_search_offset"` - MaxExportLimit int32 `yaml:"max_export_limit"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` + MaxSearchLimit int32 `yaml:"max_search_limit,omitempty"` + MaxSearchTotal int64 `yaml:"max_search_total,omitempty"` + MaxSearchOffset int32 `yaml:"max_search_offset,omitempty"` + MaxExportLimit int32 `yaml:"max_export_limit,omitempty"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit,omitempty"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests,omitempty"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request,omitempty"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts,omitempty"` } type SeqAPICaches struct { - Events *HandlerCache `yaml:"events"` - LogsLifespan *HandlerCache `yaml:"logs_lifespan"` - Fields *HandlerCache `yaml:"fields"` + Events *HandlerCache `yaml:"events,omitempty"` + LogsLifespan *HandlerCache `yaml:"logs_lifespan,omitempty"` + Fields *HandlerCache `yaml:"fields,omitempty"` } type HandlerCache struct { - ID string `yaml:"id"` - TTL time.Duration `yaml:"ttl"` + ID string `yaml:"id"` + TTL *time.Duration `yaml:"ttl,omitempty"` } type Masking struct { Masks []Mask `yaml:"masks"` - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` + ProcessFields []string `yaml:"process_fields,omitempty"` + IgnoreFields []string `yaml:"ignore_fields,omitempty"` } type Mask struct { Re string `yaml:"re"` - Groups []int `yaml:"groups"` - Mode string `yaml:"mode"` // "mask" or "replace" or "cut" - ReplaceWord string `yaml:"replace_word"` // for mode:replace + Groups []int `yaml:"groups,omitempty"` + Mode string `yaml:"mode"` // "mask" or "replace" or "cut" + ReplaceWord string `yaml:"replace_word,omitempty"` // for mode:replace - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` + ProcessFields []string `yaml:"process_fields,omitempty"` + IgnoreFields []string `yaml:"ignore_fields,omitempty"` - FieldFilters *FieldFilterSet `yaml:"field_filters"` + FieldFilters *FieldFilterSet `yaml:"field_filters,omitempty"` } type FieldFilter struct { @@ -599,39 +600,39 @@ type FieldFilterSet struct { } type LogTagsMapping struct { - Release []string `yaml:"release"` - Service []string `yaml:"service"` - Env []string `yaml:"env"` + Release []string `yaml:"release,omitempty"` + Service []string `yaml:"service,omitempty"` + Env []string `yaml:"env,omitempty"` } type ErrorGroups struct { Options ErrorGroupsOptions `yaml:"options"` - Envs map[string]ErrorGroupsEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` + Envs map[string]ErrorGroupsEnv `yaml:"envs,omitempty"` + DefaultEnv string `yaml:"default_env,omitempty"` } type ErrorGroupsEnv struct { CHID string `yaml:"ch_id"` - Options *ErrorGroupsOptions `yaml:"options"` + Options *ErrorGroupsOptions `yaml:"options,omitempty"` } type ErrorGroupsOptions struct { LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` - QueryFilter map[string]string `yaml:"query_filter"` + QueryFilter map[string]string `yaml:"query_filter,omitempty"` } type AsyncSearch struct { Options AsyncSearchOptions `yaml:"options"` - Envs map[string]AsyncSearchEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` + Envs map[string]AsyncSearchEnv `yaml:"envs,omitempty"` + DefaultEnv string `yaml:"default_env,omitempty"` } type AsyncSearchEnv struct { SeqDBID string `yaml:"seq_db_id"` - Options *AsyncSearchOptions `yaml:"options"` + Options *AsyncSearchOptions `yaml:"options,omitempty"` } type AsyncSearchOptions struct { - AdminUsers []string `yaml:"admin_users"` - ListQueryLengthLimit int `yaml:"list_query_length_limit"` + AdminUsers []string `yaml:"admin_users,omitempty"` + ListQueryLengthLimit int `yaml:"list_query_length_limit,omitempty"` } diff --git a/internal/app/config/v2/validate.go b/internal/app/config/v2/validate.go index aacc20b..c68ce95 100644 --- a/internal/app/config/v2/validate.go +++ b/internal/app/config/v2/validate.go @@ -100,7 +100,7 @@ func (cfg *Config) Validate() error { if err := validateErrorGroups(cfg.Handlers.ErrorGroups, chIDs); err != nil { return err } - if err := validateAdmin(cfg.Handlers.Admin, &cfg.Cache); err != nil { + if err := validateAdmin(cfg.Handlers.Admin, cacheIDs); err != nil { return err } @@ -309,14 +309,14 @@ func validateSeqAPI(seqAPI *SeqAPI, seqDBIDs, cacheIDs map[string]struct{}) erro return nil } -func validateAdmin(admin *Admin, cache *Cache) error { +func validateAdmin(admin *Admin, cacheIDs map[string]struct{}) error { if admin == nil { return nil } if err := validateEnvs("handlers.admin", admin.Envs, admin.DefaultEnv); err != nil { return err } - if err := validateAdminOptions("handlers.admin.options", &admin.Options, cache); err != nil { + if err := validateHandlerCache("handlers.admin.options.cache", admin.Options.Cache, cacheIDs, defaultAdminCacheTTL); err != nil { return err } @@ -325,7 +325,8 @@ func validateAdmin(admin *Admin, cache *Cache) error { envConfig.Options = &AdminOptions{} admin.Envs[envName] = envConfig } - if err := validateAdminOptions(fmt.Sprintf("handlers.admin.envs[%q].options", envName), envConfig.Options, cache); err != nil { + + if err := validateHandlerCache(fmt.Sprintf("handlers.admin.envs[%q].options.cache", envName), envConfig.Options.Cache, cacheIDs, defaultAdminCacheTTL); err != nil { return err } } @@ -333,22 +334,6 @@ func validateAdmin(admin *Admin, cache *Cache) error { return nil } -func validateAdminOptions(path string, options *AdminOptions, cache *Cache) error { - if options.Cache == nil { - return nil - } - - redis := cache.RedisByID(options.Cache.ID) - if redis == nil || redis.WithInmemID != "" { - return fmt.Errorf("%s.cache_id %q must be cache.redis without with_inmem_id", path, options.Cache.ID) - } - if options.Cache.TTL <= 0 { - options.Cache.TTL = defaultAdminCacheTTL - } - - return nil -} - func validateEnvs[T any](path string, envs map[string]T, defaultEnv string) error { if len(envs) == 0 { return nil @@ -437,8 +422,10 @@ func validateHandlerCache(path string, hc *HandlerCache, cacheIDs map[string]str if _, ok := cacheIDs[hc.ID]; !ok { return fmt.Errorf("%s.id %q not found in cache", path, hc.ID) } - if hc.TTL <= 0 { - hc.TTL = defaultTTL + if hc.TTL == nil { + hc.TTL = new(defaultTTL) + } else if *hc.TTL < 0 { + return fmt.Errorf("%s.ttl must be >= 0", path) } return nil @@ -448,11 +435,12 @@ func validateErrorGroups(errorGroups *ErrorGroups, chIDs map[string]struct{}) er if errorGroups == nil { return nil } - if err := validateEnvs("handlers.error_groups", errorGroups.Envs, errorGroups.DefaultEnv); err != nil { return err } - + if len(errorGroups.Envs) == 0 && len(chIDs) != 1 { + return fmt.Errorf("handlers.error_groups: clients must contain one client when envs are not used") + } for envName, envConfig := range errorGroups.Envs { if envConfig.CHID == "" { return fmt.Errorf("handlers.error_groups.envs[%q].ch_id cannot be empty", envName) diff --git a/internal/app/mw/auth.go b/internal/app/mw/auth.go index bb16046..bf1c1e7 100644 --- a/internal/app/mw/auth.go +++ b/internal/app/mw/auth.go @@ -43,7 +43,7 @@ func (providers AuthProvidersByEnv) get(ctx context.Context) AuthProviders { return providers[""] } -func NewAuthProvider(ctx context.Context, options config.AuthOptions) (AuthProviders, error) { +func NewAuthProvider(ctx context.Context, options config.AuthOptions, env string) (AuthProviders, error) { authPrvds := AuthProviders{} if options.JWT != nil { @@ -54,7 +54,7 @@ func NewAuthProvider(ctx context.Context, options config.AuthOptions) (AuthProvi if options.OIDC != nil { logger.Info("initializing oidc provider") var err error - authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, options.OIDC) + authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, options.OIDC, env) if err != nil { return authPrvds, fmt.Errorf("failed to init oidc provider: %w", err) } diff --git a/internal/app/mw/env.go b/internal/app/mw/env.go index f66143e..3afe5c1 100644 --- a/internal/app/mw/env.go +++ b/internal/app/mw/env.go @@ -17,7 +17,7 @@ type handlerEnvs struct { // always use root options (empty env), env from the request is ignored for them. type EnvResolver map[string]handlerEnvs -// newEnvResolver collects envs of handlers that support envs. +// NewEnvResolver collects envs of handlers that support envs. func NewEnvResolver(handlers *config.Handlers) EnvResolver { r := EnvResolver{} diff --git a/internal/app/server/init.go b/internal/app/server/init.go index b21a3e4..9ac00fd 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -46,14 +46,14 @@ func (s *Server) prepareAuth(ctx context.Context) error { s.authPrvds = make(mw.AuthProvidersByEnv, len(s.config.Auth.Envs)+1) - provider, err := mw.NewAuthProvider(ctx, s.config.Auth.Options) + provider, err := mw.NewAuthProvider(ctx, s.config.Auth.Options, "") if err != nil { return fmt.Errorf("init auth: %w", err) } s.authPrvds[""] = provider for envName, envConfig := range s.config.Auth.Envs { - provider, err := mw.NewAuthProvider(ctx, envConfig) + provider, err := mw.NewAuthProvider(ctx, envConfig, envName) if err != nil { return fmt.Errorf("init auth for env %q: %w", envName, err) } diff --git a/internal/pkg/cache/redis.go b/internal/pkg/cache/redis.go index d7e5b11..b428978 100644 --- a/internal/pkg/cache/redis.go +++ b/internal/pkg/cache/redis.go @@ -107,7 +107,7 @@ func (c *redisCache) KeysByPattern(ctx context.Context, pattern string) ([]strin return result, nil } -func (c *redisCache) MGet(ctx context.Context, keys ...string) (any, error) { +func (c *redisCache) MGet(ctx context.Context, keys ...string) ([]any, error) { prefixedKeys := make([]string, len(keys)) for i, key := range keys { prefixedKeys[i] = c.withPrefix(key) diff --git a/internal/pkg/service/admin/permissions_test.go b/internal/pkg/service/admin/permissions_test.go index 898ab4d..79870ea 100644 --- a/internal/pkg/service/admin/permissions_test.go +++ b/internal/pkg/service/admin/permissions_test.go @@ -69,7 +69,7 @@ func TestCheckAccess(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") if tt.mockArgs != nil { cache.EXPECT(). @@ -154,7 +154,7 @@ func TestGetUserPermissions(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") cache.EXPECT(). Get(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username). @@ -223,7 +223,7 @@ func TestValidatePermissions(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") err := svc.validatePermissions(tt.perms) require.Equal(t, tt.wantErr, err != nil) diff --git a/internal/pkg/service/admin/roles_test.go b/internal/pkg/service/admin/roles_test.go index f8284ac..1f620e9 100644 --- a/internal/pkg/service/admin/roles_test.go +++ b/internal/pkg/service/admin/roles_test.go @@ -20,7 +20,7 @@ var ( adminCfg = &config.AdminOptions{ SuperUsers: []string{defaultSuperUser}, Cache: &config.HandlerCache{ - TTL: 5 * time.Minute, + TTL: new(5 * time.Minute), }, } errSomethingWrong = errors.New("something happened wrong") @@ -51,7 +51,7 @@ func setupAccessMock(ctx context.Context, repo *mock.MockAdmin, cache *mock_cach Times(1) cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+access.username, gomock.Any(), adminCfg.Cache.TTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+access.username, gomock.Any(), *adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -199,7 +199,7 @@ func TestCreateRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -347,7 +347,7 @@ func TestAddUsersToRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -466,7 +466,7 @@ func TestGetRoles(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -483,7 +483,7 @@ func TestGetRoles(t *testing.T) { if tt.mockArgs.err == nil { cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyRoles, gomock.Any(), adminCfg.Cache.TTL). + SetWithTTL(gomock.Any(), cacheKeyRoles, gomock.Any(), *adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -579,7 +579,7 @@ func TestGetRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -791,7 +791,7 @@ func TestUpdateRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -950,7 +950,7 @@ func TestDeleteRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -1107,7 +1107,7 @@ func TestDeleteUsersFromRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) diff --git a/internal/pkg/service/admin/service.go b/internal/pkg/service/admin/service.go index 25211ad..6a4ab28 100644 --- a/internal/pkg/service/admin/service.go +++ b/internal/pkg/service/admin/service.go @@ -53,7 +53,7 @@ func newService(adminRepo repository.Admin, userProfilesRepo repository.UserProf ) if cfg.Cache != nil { c, _ = cache.ByID(cfg.Cache.ID) - ttl = cfg.Cache.TTL + ttl = *cfg.Cache.TTL } return &service{ diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index 38f44c0..cc572a5 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -52,17 +52,17 @@ func New( repo repository.AsyncSearches, seqDBClients map[string]seqdb.Client, ) map[string]Service { - services := make(map[string]Service, max(1, len(cfg.Envs))) + serviceByEnv := make(map[string]Service, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { for envName, envCfg := range cfg.Envs { - services[envName] = newService(ctx, repo, seqDBClients[envCfg.SeqDBID], envCfg.Options, envName) + serviceByEnv[envName] = newService(ctx, repo, seqDBClients[envCfg.SeqDBID], envCfg.Options, envName) } } else { client := slices.Collect(maps.Values(seqDBClients))[0] - services[""] = newService(ctx, repo, client, &cfg.Options, "") + serviceByEnv[""] = newService(ctx, repo, client, &cfg.Options, "") } - return services + return serviceByEnv } func newService(ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, cfg *config.AsyncSearchOptions, env string) Service { diff --git a/internal/pkg/service/errorgroups/service.go b/internal/pkg/service/errorgroups/service.go index e21c5f7..29cd5d0 100644 --- a/internal/pkg/service/errorgroups/service.go +++ b/internal/pkg/service/errorgroups/service.go @@ -41,19 +41,19 @@ type service struct { } func New(cfg *config.ErrorGroups, clientsCfg *config.Clients, chClients map[string]driver.Conn) map[string]Service { - services := make(map[string]Service, max(1, len(cfg.Envs))) + serviceByEnv := make(map[string]Service, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { for envName, envCfg := range cfg.Envs { repo := repositorych.New(chClients[envCfg.CHID], clientsCfg.ClickHouseByID(envCfg.CHID).Sharded, envCfg.Options.QueryFilter) - services[envName] = newService(repo, envCfg.Options.LogTagsMapping) + serviceByEnv[envName] = newService(repo, envCfg.Options.LogTagsMapping) } } else { chID := slices.Collect(maps.Keys(chClients))[0] repo := repositorych.New(chClients[chID], clientsCfg.ClickHouseByID(chID).Sharded, cfg.Options.QueryFilter) - services[""] = newService(repo, cfg.Options.LogTagsMapping) + serviceByEnv[""] = newService(repo, cfg.Options.LogTagsMapping) } - return services + return serviceByEnv } func newService(repo repositorych.Repository, logTagsMapping config.LogTagsMapping) Service { diff --git a/internal/pkg/service/massexport/service.go b/internal/pkg/service/massexport/service.go index 72e8223..8c72645 100644 --- a/internal/pkg/service/massexport/service.go +++ b/internal/pkg/service/massexport/service.go @@ -52,29 +52,29 @@ const ( ) func New(ctx context.Context, cfg *config.MassExport, seqDBClients map[string]seqdb.Client) (map[string]Service, error) { - services := make(map[string]Service, max(1, len(cfg.Envs))) + serviceByEnv := make(map[string]Service, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { for envName, envCfg := range cfg.Envs { - svc, err := newService(ctx, envCfg.Options, seqDBClients[envCfg.SeqDBID]) + svc, err := newService(ctx, envCfg.Options, seqDBClients[envCfg.SeqDBID], envName) if err != nil { return nil, fmt.Errorf("env %q: %w", envName, err) } - services[envName] = svc + serviceByEnv[envName] = svc } } else { client := slices.Collect(maps.Values(seqDBClients))[0] - svc, err := newService(ctx, &cfg.Options, client) + svc, err := newService(ctx, &cfg.Options, client, "") if err != nil { return nil, err } - services[""] = svc + serviceByEnv[""] = svc } - return services, nil + return serviceByEnv, nil } -func newService(ctx context.Context, cfg *config.MassExportOptions, client seqdb.Client) (Service, error) { - sessionStore, err := sessionstore.NewRedisSessionStore(ctx, &cfg.SessionStore) +func newService(ctx context.Context, cfg *config.MassExportOptions, client seqdb.Client, env string) (Service, error) { + sessionStore, err := sessionstore.NewCacheSessionStore(ctx, &cfg.SessionStore, env) if err != nil { return nil, fmt.Errorf("init session store: %w", err) } diff --git a/internal/pkg/service/massexport/sessionstore/redis.go b/internal/pkg/service/massexport/sessionstore/redis.go index 2cfe524..ee8de4e 100644 --- a/internal/pkg/service/massexport/sessionstore/redis.go +++ b/internal/pkg/service/massexport/sessionstore/redis.go @@ -16,9 +16,10 @@ import ( "github.com/ozontech/seq-ui/logger" ) -type sessionStore struct { - client cache.Store - lifetime time.Duration +type cacheSessionStore struct { + cache cache.Store + lifetime time.Duration + keyPrefix string } const ( @@ -26,15 +27,21 @@ const ( exportsKeysPattern = "job#*#export#*" ) -func NewRedisSessionStore(ctx context.Context, cfg *config.SessionStore) (SessionStore, error) { - client, ok := cache.StoreByID(cfg.CacheID) +func NewCacheSessionStore(ctx context.Context, cfg *config.SessionStore, env string) (SessionStore, error) { + c, ok := cache.StoreByID(cfg.CacheID) if !ok { return nil, fmt.Errorf("cache %q not found", cfg.CacheID) } - store := &sessionStore{ - client: client, - lifetime: cfg.Lifetime, + var keyPrefix string + if env != "" { + keyPrefix = env + "_" + } + + store := &cacheSessionStore{ + cache: c, + keyPrefix: keyPrefix, + lifetime: cfg.Lifetime, } store.unlock(ctx) @@ -42,7 +49,7 @@ func NewRedisSessionStore(ctx context.Context, cfg *config.SessionStore) (Sessio return store, nil } -func (s *sessionStore) StartExport(ctx context.Context, sessionID string, info types.ExportInfo) error { +func (s *cacheSessionStore) StartExport(ctx context.Context, sessionID string, info types.ExportInfo) error { err := s.Lock(ctx, sessionID) if err != nil { return err @@ -51,12 +58,12 @@ func (s *sessionStore) StartExport(ctx context.Context, sessionID string, info t return s.set(ctx, sessionID, info) } -func (s *sessionStore) CheckExport(ctx context.Context, sessionID string) (types.ExportInfo, error) { +func (s *cacheSessionStore) CheckExport(ctx context.Context, sessionID string) (types.ExportInfo, error) { return s.get(ctx, sessionID) } -func (s *sessionStore) GetCurActiveExport(ctx context.Context) (string, error) { - activeSessionID, err := s.client.Get(ctx, activeSessionIDKey) +func (s *cacheSessionStore) GetCurActiveExport(ctx context.Context) (string, error) { + activeSessionID, err := s.cache.Get(ctx, s.keyPrefix+activeSessionIDKey) if err == nil { return activeSessionID, nil } @@ -68,32 +75,32 @@ func (s *sessionStore) GetCurActiveExport(ctx context.Context) (string, error) { return "", fmt.Errorf("get: %w", err) } -func (s *sessionStore) CancelExport(ctx context.Context, sessionID string) error { +func (s *cacheSessionStore) CancelExport(ctx context.Context, sessionID string) error { s.unlock(ctx) return s.setExportStatus(ctx, sessionID, types.ExportStatusCancel, "") } -func (s *sessionStore) FailExport(ctx context.Context, sessionID string, errMsg string) error { +func (s *cacheSessionStore) FailExport(ctx context.Context, sessionID string, errMsg string) error { s.unlock(ctx) return s.setExportStatus(ctx, sessionID, types.ExportStatusFail, errMsg) } -func (s *sessionStore) FinishExport(ctx context.Context, sessionID string) error { +func (s *cacheSessionStore) FinishExport(ctx context.Context, sessionID string) error { s.unlock(ctx) return s.setExportStatus(ctx, sessionID, types.ExportStatusFinish, "") } -func (s *sessionStore) Lock(ctx context.Context, sessionID string) error { - ok, err := s.client.SetNX(ctx, activeSessionIDKey, sessionID, s.lifetime) +func (s *cacheSessionStore) Lock(ctx context.Context, sessionID string) error { + ok, err := s.cache.SetNX(ctx, s.keyPrefix+activeSessionIDKey, sessionID, s.lifetime) if err != nil { return fmt.Errorf("set: %w", err) } if !ok { - activeSessionID, err := s.client.Get(ctx, activeSessionIDKey) + activeSessionID, err := s.cache.Get(ctx, s.keyPrefix+activeSessionIDKey) if err != nil { return fmt.Errorf("get active session id: %w", err) } @@ -104,8 +111,8 @@ func (s *sessionStore) Lock(ctx context.Context, sessionID string) error { return nil } -func (s *sessionStore) GetAllExports(ctx context.Context) ([]types.ExportInfo, error) { - keys, err := s.client.KeysByPattern(ctx, exportsKeysPattern) +func (s *cacheSessionStore) GetAllExports(ctx context.Context) ([]types.ExportInfo, error) { + keys, err := s.cache.KeysByPattern(ctx, s.keyPrefix+exportsKeysPattern) if err != nil { return nil, fmt.Errorf("get user exports: %w", err) } @@ -115,7 +122,7 @@ func (s *sessionStore) GetAllExports(ctx context.Context) ([]types.ExportInfo, e return nil, nil } - values, err := s.client.MGet(ctx, keys...) + values, err := s.cache.MGet(ctx, keys...) if err != nil { return nil, fmt.Errorf("get values: %w", err) } @@ -135,7 +142,7 @@ func (s *sessionStore) GetAllExports(ctx context.Context) ([]types.ExportInfo, e } var globalInfo types.ExportInfo - globalInfo, err = localToGlobal(localInfo, keys[i]) + globalInfo, err = localToGlobal(localInfo, strings.TrimPrefix(keys[i], s.keyPrefix)) if err != nil { logger.Error("can't convert export info", zap.Error(err)) continue @@ -147,7 +154,7 @@ func (s *sessionStore) GetAllExports(ctx context.Context) ([]types.ExportInfo, e return result, nil } -func (s *sessionStore) ConfirmPart(ctx context.Context, sessionID string, partID int, partSize types.Size) error { +func (s *cacheSessionStore) ConfirmPart(ctx context.Context, sessionID string, partID int, partSize types.Size) error { info, err := s.get(ctx, sessionID) if err != nil { return fmt.Errorf("get: %w", err) @@ -168,7 +175,7 @@ func (s *sessionStore) ConfirmPart(ctx context.Context, sessionID string, partID return s.set(ctx, sessionID, info) } -func (s *sessionStore) setExportStatus( +func (s *cacheSessionStore) setExportStatus( ctx context.Context, sessionID string, status types.ExportStatus, @@ -194,21 +201,21 @@ func (s *sessionStore) setExportStatus( } } -func (s *sessionStore) set(ctx context.Context, sessionID string, info types.ExportInfo) error { +func (s *cacheSessionStore) set(ctx context.Context, sessionID string, info types.ExportInfo) error { data, err := json.Marshal(globalToLocal(info)) if err != nil { return fmt.Errorf("pack export info: %w", err) } - if err := s.client.SetWithTTL(ctx, sessionID, string(data), s.lifetime); err != nil { + if err := s.cache.SetWithTTL(ctx, s.keyPrefix+sessionID, string(data), s.lifetime); err != nil { return fmt.Errorf("set: %w", err) } return nil } -func (s *sessionStore) get(ctx context.Context, sessionID string) (types.ExportInfo, error) { - str, err := s.client.Get(ctx, sessionID) +func (s *cacheSessionStore) get(ctx context.Context, sessionID string) (types.ExportInfo, error) { + str, err := s.cache.Get(ctx, s.keyPrefix+sessionID) if err != nil { if errors.Is(err, cache.ErrNotFound) { return types.ExportInfo{}, types.ErrNotFound @@ -230,8 +237,8 @@ func (s *sessionStore) get(ctx context.Context, sessionID string) (types.ExportI return result, nil } -func (s *sessionStore) unlock(ctx context.Context) { - s.client.Del(ctx, activeSessionIDKey) +func (s *cacheSessionStore) unlock(ctx context.Context) { + s.cache.Del(ctx, s.keyPrefix+activeSessionIDKey) } func strToBoolArray(s string) ([]bool, error) { From 35a35bfe4fb7cc57d47a9a811b67d00cf6ad0f3b Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 21 Oct 2026 01:46:09 +0300 Subject: [PATCH 28/29] several fix --- internal/app/config/migrate/v1_v2.go | 12 +++++------- internal/app/config/v2/config.go | 4 ++-- internal/app/config/v2/validate.go | 4 ++-- internal/app/types/admin.go | 1 - internal/pkg/cache/redis.go | 2 +- internal/pkg/repository/admin_repo.go | 2 +- internal/pkg/service/admin/permissions_test.go | 4 ++-- .../pkg/service/massexport/sessionstore/redis.go | 4 ++-- swagger/swagger.json | 6 ++++-- 9 files changed, 19 insertions(+), 20 deletions(-) diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index a8de4fb..149f2ab 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -30,8 +30,8 @@ func (m *v1tov2) migrate() v2.Config { } var v1SessionStore *v1.SessionStore - if m.src.Handlers != nil && m.src.Handlers.MassExport != nil { - v1SessionStore = m.src.Handlers.MassExport.SessionStore + if handlers.MassExport != nil { + v1SessionStore = handlers.MassExport.SessionStore } return v2.Config{ @@ -300,15 +300,13 @@ func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { } func (m *v1tov2) migrateHandlers(src *v1.Handlers, server *v1.Server) v2.Handlers { - dst := v2.Handlers{ + return v2.Handlers{ SeqAPI: m.migrateSeqAPI(&src.SeqAPI, server.Cache), AsyncSearch: m.migrateAsyncSearch(&src.AsyncSearch, &src.SeqAPI), ErrorGroups: m.migrateErrorGroups(&src.ErrorGroups, server.CH), Admin: m.migrateAdmin(src.Admin, server.Cache), MassExport: m.migrateMassExport(src.MassExport, &src.SeqAPI), } - - return dst } func (m *v1tov2) migrateSeqAPI(src *v1.SeqAPI, v1Cache v1.Cache) v2.SeqAPI { @@ -513,8 +511,8 @@ func (m *v1tov2) migrateMassExport(src *v1.MassExport, seqAPI *v1.SeqAPI) *v2.Ma if src.SessionStore != nil { options.SessionStore = v2.SessionStore{ - Lifetime: src.SessionStore.ExportLifetime, - CacheID: v2.DefaultMassExportRedisID, + Lifetime: src.SessionStore.ExportLifetime, + RedisCacheID: v2.DefaultMassExportRedisID, } } diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 96c4b85..b8034e3 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -360,8 +360,8 @@ type DownloadParams struct { } type SessionStore struct { - CacheID string `yaml:"cache_id"` - Lifetime time.Duration `yaml:"lifetime,omitempty"` + RedisCacheID string `yaml:"redis_cache_id"` + Lifetime time.Duration `yaml:"lifetime,omitempty"` } type FileStore struct { diff --git a/internal/app/config/v2/validate.go b/internal/app/config/v2/validate.go index c68ce95..8a8b987 100644 --- a/internal/app/config/v2/validate.go +++ b/internal/app/config/v2/validate.go @@ -492,9 +492,9 @@ func validateMassExportOptions(path string, options *MassExportOptions, cache *C return fmt.Errorf("%s.file_store.s3 cannot be empty", path) } - redis := cache.RedisByID(options.SessionStore.CacheID) + redis := cache.RedisByID(options.SessionStore.RedisCacheID) if redis == nil || redis.WithInmemID != "" { - return fmt.Errorf("%s.session_store.cache_id %q must be cache.redis without with_inmem_id", path, options.SessionStore.CacheID) + return fmt.Errorf("%s.session_store.cache_id %q must be cache.redis without with_inmem_id", path, options.SessionStore.RedisCacheID) } setSessionStoreDefaults(&options.SessionStore, env) diff --git a/internal/app/types/admin.go b/internal/app/types/admin.go index 0cd8b32..59fe3f7 100644 --- a/internal/app/types/admin.go +++ b/internal/app/types/admin.go @@ -52,7 +52,6 @@ type UpdateRoleRequest struct { type DeleteRoleRequest struct { RoleID int32 ReplacementRoleID *int32 - Env string } type GetUserPermissionsRequest struct { diff --git a/internal/pkg/cache/redis.go b/internal/pkg/cache/redis.go index b428978..403d023 100644 --- a/internal/pkg/cache/redis.go +++ b/internal/pkg/cache/redis.go @@ -113,7 +113,7 @@ func (c *redisCache) MGet(ctx context.Context, keys ...string) ([]any, error) { prefixedKeys[i] = c.withPrefix(key) } - return c.client.MGet(ctx, keys...).Result() + return c.client.MGet(ctx, prefixedKeys...).Result() } func (c *redisCache) withPrefix(key string) string { diff --git a/internal/pkg/repository/admin_repo.go b/internal/pkg/repository/admin_repo.go index 12b48f3..ff48b52 100644 --- a/internal/pkg/repository/admin_repo.go +++ b/internal/pkg/repository/admin_repo.go @@ -209,7 +209,7 @@ func (r *adminRepository) GetUserPermissions(ctx context.Context, req types.GetU JOIN users_roles ur ON up.id=ur.user_id JOIN roles_permissions rp ON ur.role_id=rp.role_id JOIN permissions p ON rp.permission_id=p.id - WHERE ur.env=$1 up.user_name=$2 ORDER BY p.value`, []any{req.Env, req.Username} + WHERE ur.env=$1 AND up.user_name=$2 ORDER BY p.value`, []any{req.Env, req.Username} rows, err := r.pool.query(ctx, metricLabels, query, args...) if err != nil { diff --git a/internal/pkg/service/admin/permissions_test.go b/internal/pkg/service/admin/permissions_test.go index 79870ea..9e13529 100644 --- a/internal/pkg/service/admin/permissions_test.go +++ b/internal/pkg/service/admin/permissions_test.go @@ -81,7 +81,7 @@ func TestCheckAccess(t *testing.T) { Return(tt.mockArgs.perms, tt.mockArgs.err). Times(1) cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.Cache.TTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), *adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -169,7 +169,7 @@ func TestGetUserPermissions(t *testing.T) { if tt.mockArgs.err == nil { cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.Cache.TTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), *adminCfg.Cache.TTL). Return(nil). Times(1) } diff --git a/internal/pkg/service/massexport/sessionstore/redis.go b/internal/pkg/service/massexport/sessionstore/redis.go index ee8de4e..0877f72 100644 --- a/internal/pkg/service/massexport/sessionstore/redis.go +++ b/internal/pkg/service/massexport/sessionstore/redis.go @@ -28,9 +28,9 @@ const ( ) func NewCacheSessionStore(ctx context.Context, cfg *config.SessionStore, env string) (SessionStore, error) { - c, ok := cache.StoreByID(cfg.CacheID) + c, ok := cache.StoreByID(cfg.RedisCacheID) if !ok { - return nil, fmt.Errorf("cache %q not found", cfg.CacheID) + return nil, fmt.Errorf("cache %q not found", cfg.RedisCacheID) } var keyPrefix string diff --git a/swagger/swagger.json b/swagger/swagger.json index a812ae0..1f5131b 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -3706,10 +3706,12 @@ } }, "limit": { - "type": "integer" + "type": "integer", + "format": "int32" }, "offset": { - "type": "integer" + "type": "integer", + "format": "int32" }, "offset_id": { "type": "string" From 33bdd1732edbc565141e4434abc9b0493d00b68b Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 21 Oct 2026 12:41:26 +0300 Subject: [PATCH 29/29] revert ttl logic --- internal/api/seqapi/v1/grpc/events_test.go | 4 ++-- internal/api/seqapi/v1/grpc/fields_test.go | 2 +- .../api/seqapi/v1/grpc/logs_lifespan_test.go | 2 +- internal/api/seqapi/v1/grpc/test_data.go | 6 ++--- .../seqapi/v1/handler_cache/handler_cache.go | 2 +- internal/api/seqapi/v1/http/events_test.go | 4 ++-- internal/api/seqapi/v1/http/fields_test.go | 2 +- .../api/seqapi/v1/http/logs_lifespan_test.go | 2 +- internal/api/seqapi/v1/http/test_data.go | 6 ++--- internal/app/config/migrate/v1_v2.go | 20 +++++++++++------ internal/app/config/v2/config.go | 4 ++-- internal/app/config/v2/validate.go | 22 ++++++------------- .../pkg/service/admin/permissions_test.go | 10 ++++----- internal/pkg/service/admin/roles_test.go | 20 ++++++++--------- internal/pkg/service/admin/service.go | 2 +- 15 files changed, 53 insertions(+), 55 deletions(-) diff --git a/internal/api/seqapi/v1/grpc/events_test.go b/internal/api/seqapi/v1/grpc/events_test.go index 8bb9956..cf84867 100644 --- a/internal/api/seqapi/v1/grpc/events_test.go +++ b/internal/api/seqapi/v1/grpc/events_test.go @@ -106,7 +106,7 @@ func TestGetEvent(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Events: &config.HandlerCache{TTL: new(cacheTTL)}, + Events: &config.HandlerCache{TTL: cacheTTL}, }, }, }, @@ -373,7 +373,7 @@ func TestGetEventWithMasking(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Events: &config.HandlerCache{TTL: new(cacheTTL)}, + Events: &config.HandlerCache{TTL: cacheTTL}, }, Masking: tt.maskingCfg, }, diff --git a/internal/api/seqapi/v1/grpc/fields_test.go b/internal/api/seqapi/v1/grpc/fields_test.go index 9effbda..0cdc9a6 100644 --- a/internal/api/seqapi/v1/grpc/fields_test.go +++ b/internal/api/seqapi/v1/grpc/fields_test.go @@ -186,7 +186,7 @@ func TestGetFieldsCached(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Fields: &config.HandlerCache{TTL: new(ttl)}, + Fields: &config.HandlerCache{TTL: ttl}, }, }, }, diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index c93dfb9..e71d99f 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -108,7 +108,7 @@ func TestGetLogsLifespan(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - LogsLifespan: &config.HandlerCache{TTL: new(cacheTTL)}, + LogsLifespan: &config.HandlerCache{TTL: cacheTTL}, }, }, }, diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index 963a77a..a7caef0 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -41,13 +41,13 @@ func setupTestAPI(data test.APITestData) *API { caches := data.Cfg.Options.Caches for env, params := range api.paramsByEnv { if caches.Events != nil { - params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: *caches.Events.TTL} + params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: caches.Events.TTL} } if caches.Fields != nil { - params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: *caches.Fields.TTL} + params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: caches.Fields.TTL} } if caches.LogsLifespan != nil { - params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: *caches.LogsLifespan.TTL} + params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: caches.LogsLifespan.TTL} } api.paramsByEnv[env] = params } diff --git a/internal/api/seqapi/v1/handler_cache/handler_cache.go b/internal/api/seqapi/v1/handler_cache/handler_cache.go index b1a26a7..6c0ec1a 100644 --- a/internal/api/seqapi/v1/handler_cache/handler_cache.go +++ b/internal/api/seqapi/v1/handler_cache/handler_cache.go @@ -47,7 +47,7 @@ func newCache(cfg *config.HandlerCache, keyPrefix string) *Cache { return nil } - return &Cache{Cache: c, KeyPrefix: keyPrefix, TTL: *cfg.TTL} + return &Cache{Cache: c, KeyPrefix: keyPrefix, TTL: cfg.TTL} } func (c *Cache) Get(ctx context.Context, key string) (string, bool) { diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index b679f88..7682771 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -122,7 +122,7 @@ func TestServeGetEvent(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Events: &config.HandlerCache{TTL: new(cacheTTL)}, + Events: &config.HandlerCache{TTL: cacheTTL}, }, }, }, @@ -384,7 +384,7 @@ func TestGetEventWithMasking(t *testing.T) { Cfg: config.SeqAPI{ Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ - Events: &config.HandlerCache{TTL: new(cacheTTL)}, + Events: &config.HandlerCache{TTL: cacheTTL}, }, Masking: tt.maskingCfg, }, diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index 274c55e..a8c5de4 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -97,7 +97,7 @@ func TestServeGetFields(t *testing.T) { cfg: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ Fields: &config.HandlerCache{ - TTL: new(time.Hour), + TTL: time.Hour, }, }, }, diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index fa018de..616cce6 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -106,7 +106,7 @@ func TestServeGetLogsLifespan(t *testing.T) { Options: config.SeqAPIOptions{ Caches: config.SeqAPICaches{ LogsLifespan: &config.HandlerCache{ - TTL: new(cacheTTL), + TTL: cacheTTL, }, }, }, diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 5661077..17d5f5d 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -46,13 +46,13 @@ func setupTestAPI(data test.APITestData) *API { caches := data.Cfg.Options.Caches for env, params := range api.paramsByEnv { if caches.Events != nil { - params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: *caches.Events.TTL} + params.caches.Events = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_event_", TTL: caches.Events.TTL} } if caches.Fields != nil { - params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: *caches.Fields.TTL} + params.caches.Fields = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_fields", TTL: caches.Fields.TTL} } if caches.LogsLifespan != nil { - params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: *caches.LogsLifespan.TTL} + params.caches.LogsLifespan = &handlercache.Cache{Cache: data.Mocks.Cache, KeyPrefix: "seqapi_logs_lifespan", TTL: caches.LogsLifespan.TTL} } api.paramsByEnv[env] = params } diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index 149f2ab..5ad5eb3 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -7,6 +7,12 @@ import ( v2 "github.com/ozontech/seq-ui/internal/app/config/v2" ) +const ( + v1DefaultEventsCacheTTL = 24 * time.Hour + v1DefaultLogsLifespanCacheTTL = 10 * time.Minute + v1DefaultAdminCacheTTL = time.Hour +) + type v1tov2 struct { src v1.Config } @@ -355,11 +361,11 @@ func (m *v1tov2) migrateSeqAPIOptions(src *v1.SeqAPIOptions, v1Cache v1.Cache) v Caches: v2.SeqAPICaches{ Events: &v2.HandlerCache{ ID: redisWithInmemID, - TTL: m.migrateTTL(src.EventsCacheTTL), + TTL: m.migrateTTL(src.EventsCacheTTL, v1DefaultEventsCacheTTL), }, LogsLifespan: &v2.HandlerCache{ ID: redisOrInmemID, - TTL: m.migrateTTL(src.LogsLifespanCacheTTL), + TTL: m.migrateTTL(src.LogsLifespanCacheTTL, v1DefaultLogsLifespanCacheTTL), }, }, } @@ -368,7 +374,7 @@ func (m *v1tov2) migrateSeqAPIOptions(src *v1.SeqAPIOptions, v1Cache v1.Cache) v if src.FieldsCacheTTL > 0 { dst.Caches.Fields = &v2.HandlerCache{ ID: v2.DefaultInmemID, - TTL: m.migrateTTL(src.FieldsCacheTTL), + TTL: src.FieldsCacheTTL, } } @@ -552,18 +558,18 @@ func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { if v1Cache.Redis != nil { dst.Options.Cache = &v2.HandlerCache{ ID: v2.DefaultRedisWithoutInmemID, - TTL: m.migrateTTL(src.CacheTTL), + TTL: m.migrateTTL(src.CacheTTL, v1DefaultAdminCacheTTL), } } return dst } -func (m *v1tov2) migrateTTL(ttl time.Duration) *time.Duration { +func (m *v1tov2) migrateTTL(ttl time.Duration, defaultTTL time.Duration) time.Duration { if ttl <= 0 { - return nil + return defaultTTL } - return &ttl + return ttl } // defaultCacheIDs returns cache ids same as in v1. diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index b8034e3..d19dfce 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -566,8 +566,8 @@ type SeqAPICaches struct { } type HandlerCache struct { - ID string `yaml:"id"` - TTL *time.Duration `yaml:"ttl,omitempty"` + ID string `yaml:"id"` + TTL time.Duration `yaml:"ttl,omitempty"` } type Masking struct { diff --git a/internal/app/config/v2/validate.go b/internal/app/config/v2/validate.go index 8a8b987..465a166 100644 --- a/internal/app/config/v2/validate.go +++ b/internal/app/config/v2/validate.go @@ -54,12 +54,6 @@ const ( defaultInmemCacheMaxCost = 1000000 defaultInmemCacheBufferItems = 64 - defaultEventsCacheTTL = 24 * time.Hour - - defaultLogsLifespanCacheTTL = 10 * time.Minute - - defaultAdminCacheTTL = time.Hour - defaultClickHouseDialTimeout = 5 * time.Second defaultClickHouseReadTimeout = 30 * time.Second @@ -316,7 +310,7 @@ func validateAdmin(admin *Admin, cacheIDs map[string]struct{}) error { if err := validateEnvs("handlers.admin", admin.Envs, admin.DefaultEnv); err != nil { return err } - if err := validateHandlerCache("handlers.admin.options.cache", admin.Options.Cache, cacheIDs, defaultAdminCacheTTL); err != nil { + if err := validateHandlerCache("handlers.admin.options.cache", admin.Options.Cache, cacheIDs); err != nil { return err } @@ -326,7 +320,7 @@ func validateAdmin(admin *Admin, cacheIDs map[string]struct{}) error { admin.Envs[envName] = envConfig } - if err := validateHandlerCache(fmt.Sprintf("handlers.admin.envs[%q].options.cache", envName), envConfig.Options.Cache, cacheIDs, defaultAdminCacheTTL); err != nil { + if err := validateHandlerCache(fmt.Sprintf("handlers.admin.envs[%q].options.cache", envName), envConfig.Options.Cache, cacheIDs); err != nil { return err } } @@ -405,26 +399,24 @@ func validateSeqAPIOptions(path string, options *SeqAPIOptions, cacheIDs map[str } caches := &options.Caches - if err := validateHandlerCache(path+".caches.events", caches.Events, cacheIDs, defaultEventsCacheTTL); err != nil { + if err := validateHandlerCache(path+".caches.events", caches.Events, cacheIDs); err != nil { return err } - if err := validateHandlerCache(path+".caches.logs_lifespan", caches.LogsLifespan, cacheIDs, defaultLogsLifespanCacheTTL); err != nil { + if err := validateHandlerCache(path+".caches.logs_lifespan", caches.LogsLifespan, cacheIDs); err != nil { return err } - return validateHandlerCache(path+".caches.fields", caches.Fields, cacheIDs, 0) + return validateHandlerCache(path+".caches.fields", caches.Fields, cacheIDs) } -func validateHandlerCache(path string, hc *HandlerCache, cacheIDs map[string]struct{}, defaultTTL time.Duration) error { +func validateHandlerCache(path string, hc *HandlerCache, cacheIDs map[string]struct{}) error { if hc == nil { return nil } if _, ok := cacheIDs[hc.ID]; !ok { return fmt.Errorf("%s.id %q not found in cache", path, hc.ID) } - if hc.TTL == nil { - hc.TTL = new(defaultTTL) - } else if *hc.TTL < 0 { + if hc.TTL < 0 { return fmt.Errorf("%s.ttl must be >= 0", path) } diff --git a/internal/pkg/service/admin/permissions_test.go b/internal/pkg/service/admin/permissions_test.go index 9e13529..898ab4d 100644 --- a/internal/pkg/service/admin/permissions_test.go +++ b/internal/pkg/service/admin/permissions_test.go @@ -69,7 +69,7 @@ func TestCheckAccess(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") if tt.mockArgs != nil { cache.EXPECT(). @@ -81,7 +81,7 @@ func TestCheckAccess(t *testing.T) { Return(tt.mockArgs.perms, tt.mockArgs.err). Times(1) cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), *adminCfg.Cache.TTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -154,7 +154,7 @@ func TestGetUserPermissions(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") cache.EXPECT(). Get(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username). @@ -169,7 +169,7 @@ func TestGetUserPermissions(t *testing.T) { if tt.mockArgs.err == nil { cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), *adminCfg.Cache.TTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -223,7 +223,7 @@ func TestValidatePermissions(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") err := svc.validatePermissions(tt.perms) require.Equal(t, tt.wantErr, err != nil) diff --git a/internal/pkg/service/admin/roles_test.go b/internal/pkg/service/admin/roles_test.go index 1f620e9..f8284ac 100644 --- a/internal/pkg/service/admin/roles_test.go +++ b/internal/pkg/service/admin/roles_test.go @@ -20,7 +20,7 @@ var ( adminCfg = &config.AdminOptions{ SuperUsers: []string{defaultSuperUser}, Cache: &config.HandlerCache{ - TTL: new(5 * time.Minute), + TTL: 5 * time.Minute, }, } errSomethingWrong = errors.New("something happened wrong") @@ -51,7 +51,7 @@ func setupAccessMock(ctx context.Context, repo *mock.MockAdmin, cache *mock_cach Times(1) cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+access.username, gomock.Any(), *adminCfg.Cache.TTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+access.username, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -199,7 +199,7 @@ func TestCreateRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -347,7 +347,7 @@ func TestAddUsersToRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -466,7 +466,7 @@ func TestGetRoles(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -483,7 +483,7 @@ func TestGetRoles(t *testing.T) { if tt.mockArgs.err == nil { cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyRoles, gomock.Any(), *adminCfg.Cache.TTL). + SetWithTTL(gomock.Any(), cacheKeyRoles, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -579,7 +579,7 @@ func TestGetRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -791,7 +791,7 @@ func TestUpdateRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -950,7 +950,7 @@ func TestDeleteRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -1107,7 +1107,7 @@ func TestDeleteUsersFromRole(t *testing.T) { upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) svc := newService(adminRepo, upRepo, adminCfg, "").(*service) - svc.cache = newAdminCache(cache, *adminCfg.Cache.TTL, "") + svc.cache = newAdminCache(cache, adminCfg.Cache.TTL, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) diff --git a/internal/pkg/service/admin/service.go b/internal/pkg/service/admin/service.go index 6a4ab28..25211ad 100644 --- a/internal/pkg/service/admin/service.go +++ b/internal/pkg/service/admin/service.go @@ -53,7 +53,7 @@ func newService(adminRepo repository.Admin, userProfilesRepo repository.UserProf ) if cfg.Cache != nil { c, _ = cache.ByID(cfg.Cache.ID) - ttl = *cfg.Cache.TTL + ttl = cfg.Cache.TTL } return &service{