diff --git a/Makefile b/Makefile index ff3d979..394f220 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ TIME := $(shell date '+%Y-%m-%d_%H:%M:%S') MIGRATION_DSN ?= postgresql://localhost/postgres?sslmode=disable&user=postgres&password=postgres MIGRATION_DSN_CLICKHOUSE ?= tcp://default@localhost:9000/seq_ui_server +SOURCE_CONFIG ?= config/config.local.yaml LOCAL_BIN := $(CURDIR)/bin @@ -58,6 +59,10 @@ push-migration-image: build-migration-image run: .check-config go run ./cmd/seq-ui -config=config/config.local.yaml +.PHONY: config-migrate +config-migrate: + go run ./cmd/config-migrate -source=${SOURCE_CONFIG} + .PHONY: .check-config .check-config: @if [ ! -f config/config.local.yaml ]; then\ diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go new file mode 100644 index 0000000..6cc2abc --- /dev/null +++ b/cmd/config-migrate/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "errors" + "flag" + "os" + "path/filepath" + "strings" + + "go.uber.org/zap" + + "github.com/ozontech/seq-ui/internal/app/config/loader" + "github.com/ozontech/seq-ui/logger" +) + +var ( + source = flag.String("source", "", "path to the config file to migrate in place") +) + +func main() { + flag.Parse() + + run(*source) +} + +func run(source string) { + if source == "" { + logger.Fatal("miss required parameter", zap.String("param", "-source")) + } + + stat, err := os.Stat(source) + if err != nil { + logger.Fatal("stat source", zap.String("source", source), zap.Error(err)) + } + + sourceCfg, err := os.ReadFile(source) //nolint:gosec + if err != nil { + logger.Fatal("read source config", zap.String("source", source), zap.Error(err)) + } + + migratedCfg, err := loader.ToLatestVersion(sourceCfg) + if errors.Is(err, loader.ErrAlreadyLatestConfigVersion) { + logger.Info("source config is already at the latest schema version, nothing to migrate", zap.String("source", source)) + return + } + if err != nil { + logger.Fatal("migrate config", zap.String("source", source), zap.Error(err)) + } + + backupPath := strings.TrimSuffix(source, filepath.Ext(source)) + ".bck" + filepath.Ext(source) + if _, err := os.Stat(backupPath); err == nil { + logger.Fatal("backup already exists, remove it before re-running", zap.String("backup", backupPath)) + } + if err := os.WriteFile(backupPath, sourceCfg, stat.Mode()); err != nil { + logger.Fatal("write backup config", zap.String("backup", backupPath), zap.Error(err)) + } + + if err := os.WriteFile(source, migratedCfg, stat.Mode()); err != nil { + logger.Fatal("write migrated config", zap.Error(err)) + } + + logger.Info("config migrated", zap.String("source", source), zap.String("backup", backupPath)) +} diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 2a50a21..e4a4d9a 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -6,7 +6,6 @@ import ( "flag" "fmt" "net/http" - "os" "os/signal" "syscall" @@ -17,198 +16,164 @@ import ( "github.com/joho/godotenv" "go.uber.org/zap" + "github.com/ozontech/seq-ui/buildinfo" "github.com/ozontech/seq-ui/internal/api" admin_v1 "github.com/ozontech/seq-ui/internal/api/admin/v1" dashboards_v1 "github.com/ozontech/seq-ui/internal/api/dashboards/v1" errorgroups_v1 "github.com/ozontech/seq-ui/internal/api/errorgroups/v1" - massexport_v1 "github.com/ozontech/seq-ui/internal/api/massexport/v1" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/loader" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/server" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/repository" - repositorych "github.com/ozontech/seq-ui/internal/pkg/repository_ch" "github.com/ozontech/seq-ui/internal/pkg/service/admin" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" "github.com/ozontech/seq-ui/internal/pkg/service/dashboards" "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" - "github.com/ozontech/seq-ui/internal/pkg/service/massexport" - "github.com/ozontech/seq-ui/internal/pkg/service/massexport/filestore" - "github.com/ozontech/seq-ui/internal/pkg/service/massexport/sessionstore" "github.com/ozontech/seq-ui/internal/pkg/service/profiles" "github.com/ozontech/seq-ui/internal/pkg/service/userprofile" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/tracing" ) -const ( - defaultConfig = "config/config.example.yaml" +const defaultConfig = "config/config.example.yaml" - defaultClientMaxRecvMsgSize = 4 * 1024 * 1024 // 4MB -) - -var ( - configPath = flag.String("config", defaultConfig, "application config") -) - -func init() { +func main() { // Load .env file for local development (optional). // OS environment variables take precedence. _ = godotenv.Load() -} -func main() { - ctx, cancel := signal.NotifyContext(context.Background(), - syscall.SIGTERM, - os.Interrupt, + configPath := flag.String("config", defaultConfig, "application config") + flag.Parse() + + logger.Info("hi, I am seq-ui", + zap.String("version", buildinfo.Version), + zap.String("build_time", buildinfo.BuildTime), ) + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() - run(ctx) + run(ctx, *configPath) } -func run(ctx context.Context) { - flag.Parse() - - if *configPath == defaultConfig { +func run(ctx context.Context, configPath string) { + if configPath == defaultConfig { logger.Warn("app uses the default config file, to provide your own config use -config flag") } - cfg, err := config.FromFile(*configPath) + cfg, err := loader.FromFile(configPath) if err != nil { - logger.Fatal("read config file error", zap.Error(err)) + logger.Fatal("failed to read config", zap.Error(err)) } - if tracingCfg, err := tracing.Initialize(); err != nil { - logger.Error("tracing initialization failed", zap.Error(err)) - } else { - logger.Info( - "tracing initialization success", - zap.String("service_name", tracingCfg.ServiceName), - zap.String("agent_host", tracingCfg.AgentHost), - zap.String("agent_port", tracingCfg.AgentPort), - zap.Float64("sampler_param", tracingCfg.SamplerParam)) + if err := tracing.Init(); err != nil { + logger.Error("failed to init tracing", zap.Error(err)) } registrar := initApp(ctx, cfg) - serv, err := server.New(ctx, cfg.Server, registrar) + serv, err := server.New(ctx, cfg, registrar) if err != nil { - logger.Fatal("app init error", zap.Error(err)) + logger.Fatal("failed to init servers", zap.Error(err)) } - // Run launches both grpc and http servers. On successful - // http.ErrServerClosed is returned because of http.Server.Serve. + // Run launches configured grpc/http servers and debug server. On successful + // shutdown http.ErrServerClosed is returned because of http.Server.Serve. if err = serv.Run(ctx); !errors.Is(err, http.ErrServerClosed) { - logger.Fatal("app run", zap.Error(err)) + logger.Fatal("failed to run servers", zap.Error(err)) } } -func initApp(ctx context.Context, cfg config.Config) *api.Registrar { - logger.Info("initializing seq-db clients") - seqDBClients, err := initSeqDBClients(ctx, cfg) - if err != nil { - logger.Fatal("failed to init seq-db client", zap.Error(err)) - } - - defaultClientID := config.DefaultSeqDBClientID - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - defaultClientID = cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv].SeqDB - } - - defaultClient, exists := seqDBClients[defaultClientID] - if !exists { - logger.Fatal("default seq-db client not found", - zap.String("clientID", defaultClientID), - ) - } - - var massExportV1 *massexport_v1.MassExport - if cfg.Handlers.MassExport != nil { - exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, defaultClient) - if err != nil { - logger.Fatal("can't init export server", zap.Error(err)) - } - - massExportV1 = massexport_v1.New(exportServer) +func initApp(ctx context.Context, cfg *config.Config) *api.Registrar { + logger.Info("initializing caches") + if err := cache.Init(ctx, &cfg.Cache); err != nil { + logger.Fatal("failed to init caches", zap.Error(err)) } - logger.Info("initializing inmemory with redis seqapi cache") - inmemWithRedisCache, err := cache.NewInmemoryWithRedisOrInmemory(ctx, cfg.Server.Cache) + logger.Info("initializing seq-db clients") + seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { - logger.Fatal("failed to init inmemory with redis seqapi cache", zap.Error(err)) + logger.Fatal("failed to init seq-db clients", zap.Error(err)) } - logger.Info("initializing redis seqapi cache") - redisCache, err := cache.NewRedisOrInmemory(ctx, cfg.Server.Cache) + logger.Info("initializing clickhouse clients") + chClients, err := initClickHouseClients(ctx, cfg) if err != nil { - logger.Fatal("failed to init redis seqapi cache", zap.Error(err)) + logger.Fatal("failed to init clickhouse clients", zap.Error(err)) } logger.Info("initializing db") - db, err := initDb(ctx, cfg.Server.DB) + db, err := initDb(ctx, cfg.DB) if err != nil { logger.Fatal("failed to init db", zap.Error(err)) } + // var massExportV1 *massexport_v1.MassExport + // if cfg.Handlers.MassExport != nil { + // logger.Info("initializing mass export") + // exportServer, err := initExportService(ctx, cfg.Handlers.MassExport, seqDBClients) + // if err != nil { + // logger.Fatal("can't init mass export server", zap.Error(err)) + // } + // massExportV1 = massexport_v1.New(exportServer) + // } + var ( - adminV1 *admin_v1.Admin - asyncSearchesService asyncsearches.Service - userProfileV1 *userprofile_v1.UserProfile - dashboardsV1 *dashboards_v1.Dashboards + adminV1 *admin_v1.Admin + userProfileV1 *userprofile_v1.UserProfile + dashboardsV1 *dashboards_v1.Dashboards + asyncSearchesRepo repository.AsyncSearches ) if db != nil { - repo := repository.New(db, cfg.Server.DB.RequestTimeout) + repo := repository.New(db, cfg.DB.RequestTimeout) userProfilesSvc := userprofile.New(repo.UserProfiles, repo.FavoriteQueries) dashboardsSvc := dashboards.New(repo.Dashboards) - profiles.InitProfiles(repo.UserProfiles.GetOrCreate) + profiles.Init(repo.UserProfiles.GetOrCreate) + logger.Info("initializing user profile") userProfileV1 = userprofile_v1.New(userProfilesSvc) + logger.Info("initializing dashboards") dashboardsV1 = dashboards_v1.New(dashboardsSvc) - asyncSearchesService = asyncsearches.New(ctx, repo, defaultClient, cfg.Handlers.AsyncSearch) + if cfg.Handlers.AsyncSearch != nil { + logger.Info("initializing async search") + asyncSearchesRepo = repo.AsyncSearches + } if cfg.Handlers.Admin != nil { - logger.Info("initializing redis admin cache") - adminCache, err := cache.NewRedis(ctx, cfg.Server.Cache.Redis) - if err != nil { - logger.Warn("failed to init redis admin cache, admin will run without cache", zap.Error(err)) - } else if adminCache == nil { - logger.Info("redis cache config is not set, admin will run without cache") - } - - adminSvc := admin.New(repo.Admin, repo.UserProfiles, adminCache, cfg.Handlers.Admin) + logger.Info("initializing admin") + adminSvc := admin.NewByEnv(cfg.Handlers.Admin, repo.Admin, repo.UserProfiles) adminV1 = admin_v1.New(adminSvc) } } - seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService) - - logger.Info("initializing clickhouse") - ch, err := initClickHouse(ctx, cfg.Server.CH) - if err != nil { - logger.Fatal("failed to init clickhouse", zap.Error(err)) + var seqApiV1 *seqapi_v1.SeqAPI + if cfg.Handlers.SeqAPI != nil { + logger.Info("initializing seqapi") + asyncSearchesSvc := asyncsearches.NewByEnv(ctx, cfg.Handlers.AsyncSearch, asyncSearchesRepo, seqDBClients) + seqApiV1 = seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, asyncSearchesSvc) } var errorGroupsV1 *errorgroups_v1.ErrorGroups - if ch != nil { - repo := repositorych.New(ch, cfg.Server.CH.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) - svc := errorgroups.New(repo, cfg.Handlers.ErrorGroups.LogTagsMapping) - - errorGroupsV1 = errorgroups_v1.New(svc) + if cfg.Handlers.ErrorGroups != nil { + logger.Info("initializing error groups") + errorGroupsSvc := errorgroups.NewByEnv(cfg.Handlers.ErrorGroups, &cfg.Clients, chClients) + errorGroupsV1 = errorgroups_v1.New(errorGroupsSvc) } return api.NewRegistrar(adminV1, seqApiV1, userProfileV1, dashboardsV1, massExportV1, errorGroupsV1) } -func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb.Client, error) { - clients := make(map[string]seqdb.Client) +func initSeqDBClients(ctx context.Context, cfg *config.Config) (map[string]seqdb.Client, error) { + clients := make(map[string]seqdb.Client, len(cfg.Clients.SeqDB)) for _, clientCfg := range cfg.Clients.SeqDB { - client, err := createSeqBDClient(ctx, clientCfg, cfg.Handlers.SeqAPI) + client, err := createSeqBDClient(ctx, &clientCfg, cfg.Handlers.SeqAPI) if err != nil { - return nil, fmt.Errorf("failed to create seq_db client %s: %w", clientCfg.ID, err) + return nil, fmt.Errorf("failed to create seq-db client %q: %w", clientCfg.ID, err) } clients[clientCfg.ID] = client } @@ -216,23 +181,36 @@ func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb. return clients, nil } -func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI config.SeqAPI) (seqdb.Client, error) { - clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * int(seqAPI.MaxSearchLimit) - clientMaxRecvMsgSize = max(clientMaxRecvMsgSize, defaultClientMaxRecvMsgSize) +func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *config.SeqAPI) (seqdb.Client, error) { + var maxSearchLimit int32 + if seqAPI != nil { + if len(seqAPI.Envs) == 0 { + maxSearchLimit = seqAPI.Options.Limits.MaxSearchLimit + } + + // one seqdb client can be used by several envs, so it must accept the largest one + for _, env := range seqAPI.Envs { + if env.SeqDBID == cfg.ID { + maxSearchLimit = max(maxSearchLimit, env.Options.Limits.MaxSearchLimit) + } + } + } clientParams := seqdb.ClientParams{ - Addrs: cfg.Addrs, - Timeout: cfg.Timeout, - MaxRetries: cfg.RequestRetries, - InitialRetryBackoff: cfg.InitialRetryBackoff, - MaxRetryBackoff: cfg.MaxRetryBackoff, - MaxRecvMsgSize: clientMaxRecvMsgSize, + Addrs: cfg.Addrs, + Timeout: cfg.Timeout, + MaxRetries: cfg.RequestRetries, + MaxRecvMsgSize: cfg.AvgDocSize * 1024 * int(maxSearchLimit), + } + if cfg.RequestRetriesBackoff != nil { + clientParams.InitialRetryBackoff = cfg.RequestRetriesBackoff.InitialInterval + clientParams.MaxRetryBackoff = cfg.RequestRetriesBackoff.MaxInterval } - if cfg.GRPCKeepaliveParams != nil { + if cfg.GRPCParams != nil && cfg.GRPCParams.Keepalive != nil { clientParams.GRPCKeepaliveParams = &seqdb.GRPCKeepaliveParams{ - Time: cfg.GRPCKeepaliveParams.Time, - Timeout: cfg.GRPCKeepaliveParams.Timeout, - PermitWithoutStream: cfg.GRPCKeepaliveParams.PermitWithoutStream, + Time: cfg.GRPCParams.Keepalive.Time, + Timeout: cfg.GRPCParams.Keepalive.Timeout, + PermitWithoutStream: cfg.GRPCParams.Keepalive.PermitWithoutStream, } } @@ -244,12 +222,10 @@ func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { logger.Warn("db config is nil, running without db") return nil, nil } - pgxCfg, err := pgxpool.ParseConfig(cfg.ConnString()) if err != nil { - return nil, fmt.Errorf("can't parse connection string: %w", err) + return nil, fmt.Errorf("failed to parse connection string: %w", err) } - if !*cfg.UsePreparedStatements { // By default, pgx uses the QueryExecModeCacheStatement and automatically prepares and caches prepared statements. // However, this may be incompatible with proxies such as PGBouncer. @@ -259,34 +235,46 @@ func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { pool, err := pgxpool.NewWithConfig(ctx, pgxCfg) if err != nil { - return nil, fmt.Errorf("can't create pgx pool: %w", err) + return nil, fmt.Errorf("failed to create pgx pool: %w", err) + } + + if err := pool.Ping(ctx); err != nil { + return nil, fmt.Errorf("failed to ping db: %w", err) } return pool, nil } -func initExportService(ctx context.Context, cfg config.MassExport, client seqdb.Client) (massexport.Service, error) { - sessionStore, err := sessionstore.NewRedisSessionStore(ctx, cfg.SessionStore) - if err != nil { - return nil, fmt.Errorf("init session store: %w", err) - } - logger.Info("session store initialized") - - fileStore, err := filestore.NewS3(ctx, cfg.FileStore.S3) - if err != nil { - return nil, fmt.Errorf("init file store: %w", err) +// func initExportService(ctx context.Context, cfg config.MassExport, redisCfg *config.Redis, client seqdb.Client) (massexport.Service, error) { +// sessionStore, err := sessionstore.NewRedisSessionStore(ctx, redisCfg, cfg.SessionStore.ExportLifetime) +// if err != nil { +// return nil, fmt.Errorf("init session store: %w", err) +// } +// logger.Info("session store initialized") +// +// fileStore, err := filestore.NewS3(ctx, cfg.FileStore.S3) +// if err != nil { +// return nil, fmt.Errorf("init file store: %w", err) +// } +// logger.Info("file store initialized") +// +// return massexport.NewService(ctx, cfg, sessionStore, fileStore, client) +// } + +func initClickHouseClients(ctx context.Context, cfg *config.Config) (map[string]driver.Conn, error) { + clients := make(map[string]driver.Conn, len(cfg.Clients.ClickHouse)) + for _, clientCfg := range cfg.Clients.ClickHouse { + client, err := createClickHouseClient(ctx, &clientCfg) + if err != nil { + return nil, fmt.Errorf("failed to create clickhouse client %q: %w", clientCfg.ID, err) + } + clients[clientCfg.ID] = client } - logger.Info("file store initialized") - return massexport.NewService(ctx, cfg, sessionStore, fileStore, client) + return clients, nil } -func initClickHouse(ctx context.Context, cfg *config.CH) (driver.Conn, error) { - if cfg == nil { - logger.Warn("clickhouse config is nil, running without clickhouse") - return nil, nil - } - +func createClickHouseClient(ctx context.Context, cfg *config.CHClient) (driver.Conn, error) { conn, err := clickhouse.Open(&clickhouse.Options{ Addr: cfg.Addrs, Auth: clickhouse.Auth{ diff --git a/internal/api/admin/v1/admin.go b/internal/api/admin/v1/admin.go index 9bcee27..ac8c549 100644 --- a/internal/api/admin/v1/admin.go +++ b/internal/api/admin/v1/admin.go @@ -13,10 +13,10 @@ type Admin struct { httpAPI *http_api.API } -func New(svc admin.Service) *Admin { +func New(services map[string]admin.Service) *Admin { return &Admin{ - grpcAPI: grpc_api.New(svc), - httpAPI: http_api.New(svc), + grpcAPI: grpc_api.New(services), + httpAPI: http_api.New(services), } } diff --git a/internal/api/admin/v1/grpc/api.go b/internal/api/admin/v1/grpc/api.go index 1368bc4..12072f2 100644 --- a/internal/api/admin/v1/grpc/api.go +++ b/internal/api/admin/v1/grpc/api.go @@ -9,14 +9,21 @@ import ( type API struct { api.UnimplementedAdminServiceServer - service admin.Service + services map[string]admin.Service availablePermissions []*api.PermissionGroup } -func New(svc admin.Service) *API { +func New(services map[string]admin.Service) *API { + // available permissions are the same for all envs + var availablePermissions []*api.PermissionGroup + for _, svc := range services { + availablePermissions = availablePermissionsToProto(svc.GetAvailablePermissions()) + break + } + return &API{ - service: svc, - availablePermissions: availablePermissionsToProto(svc.GetAvailablePermissions()), + services: services, + availablePermissions: availablePermissions, } } diff --git a/internal/api/admin/v1/grpc/roles.go b/internal/api/admin/v1/grpc/roles.go index c41e695..2e9e397 100644 --- a/internal/api/admin/v1/grpc/roles.go +++ b/internal/api/admin/v1/grpc/roles.go @@ -34,7 +34,8 @@ func (a *API) CreateRole(ctx context.Context, req *admin.CreateRoleRequest) (*ad Permissions: permissionGroupsToStrings(req.Permissions), } - roleID, err := a.service.CreateRole(ctx, request) + svc := a.services[types.GetEnvKey(ctx)] + roleID, err := svc.CreateRole(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } @@ -59,7 +60,8 @@ func (a *API) AddUsersToRole(ctx context.Context, req *admin.AddUsersToRoleReque }, ) - if err := a.service.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ RoleID: req.RoleId, Usernames: req.Usernames, }); err != nil { @@ -84,7 +86,8 @@ func (a *API) DeleteUsersFromRole(ctx context.Context, req *admin.DeleteUsersFro }, ) - if err := a.service.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ RoleID: req.RoleId, Usernames: req.Usernames, }); err != nil { @@ -95,7 +98,8 @@ func (a *API) DeleteUsersFromRole(ctx context.Context, req *admin.DeleteUsersFro } func (a *API) GetRoles(ctx context.Context, _ *admin.GetRolesRequest) (*admin.GetRolesResponse, error) { - resp, err := a.service.GetRoles(ctx) + svc := a.services[types.GetEnvKey(ctx)] + resp, err := svc.GetRoles(ctx) if err != nil { return nil, grpcutil.ProcessError(err) } @@ -117,7 +121,8 @@ func (a *API) GetRole(ctx context.Context, req *admin.GetRoleRequest) (*admin.Ge }, ) - roleInfo, err := a.service.GetRole(ctx, types.GetRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + roleInfo, err := svc.GetRole(ctx, types.GetRoleRequest{ RoleID: req.Id, }) if err != nil { @@ -151,7 +156,8 @@ func (a *API) UpdateRole(ctx context.Context, req *admin.UpdateRoleRequest) (*ad } span.SetAttributes(spanAttributes...) - if err := a.service.UpdateRole(ctx, types.UpdateRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.UpdateRole(ctx, types.UpdateRoleRequest{ RoleID: req.Id, Name: req.Name, Permissions: permissionGroupsToStrings(req.Permissions), @@ -180,7 +186,8 @@ func (a *API) DeleteRole(ctx context.Context, req *admin.DeleteRoleRequest) (*ad } span.SetAttributes(spanAttributes...) - if err := a.service.DeleteRole(ctx, types.DeleteRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.DeleteRole(ctx, types.DeleteRoleRequest{ RoleID: req.Id, ReplacementRoleID: req.ReplacementRoleId, }); err != nil { diff --git a/internal/api/admin/v1/grpc/roles_test.go b/internal/api/admin/v1/grpc/roles_test.go index d10785e..5a45f5d 100644 --- a/internal/api/admin/v1/grpc/roles_test.go +++ b/internal/api/admin/v1/grpc/roles_test.go @@ -11,6 +11,7 @@ import ( "google.golang.org/grpc/status" "github.com/ozontech/seq-ui/internal/app/types" + admin_service "github.com/ozontech/seq-ui/internal/pkg/service/admin" mock "github.com/ozontech/seq-ui/internal/pkg/service/admin/mock" "github.com/ozontech/seq-ui/pkg/admin/v1" ) @@ -33,7 +34,7 @@ func setupTestAPI(t *testing.T) (*API, *mock.MockService) { }). Times(1) - return New(mockedSvc), mockedSvc + return New(map[string]admin_service.Service{"": mockedSvc}), mockedSvc } func TestCreateRole(t *testing.T) { diff --git a/internal/api/admin/v1/http/api.go b/internal/api/admin/v1/http/api.go index b2e38a0..823a2f5 100644 --- a/internal/api/admin/v1/http/api.go +++ b/internal/api/admin/v1/http/api.go @@ -8,14 +8,21 @@ import ( ) type API struct { - service adminservice.Service + services map[string]adminservice.Service availablePermissions []permissionGroup } -func New(svc adminservice.Service) *API { +func New(services map[string]adminservice.Service) *API { + // available permissions are the same for all envs + var availablePermissions []permissionGroup + for _, svc := range services { + availablePermissions = parseAvailablePermissions(svc.GetAvailablePermissions()) + break + } + return &API{ - service: svc, - availablePermissions: parseAvailablePermissions(svc.GetAvailablePermissions()), + services: services, + availablePermissions: availablePermissions, } } diff --git a/internal/api/admin/v1/http/roles.go b/internal/api/admin/v1/http/roles.go index 04ab891..467b976 100644 --- a/internal/api/admin/v1/http/roles.go +++ b/internal/api/admin/v1/http/roles.go @@ -24,6 +24,7 @@ const permSep = ":" // @Router /admin/v1/roles [post] // @ID admin_v1_create_role // @Tags admin_v1 +// @Param env query string false "Environment" // @Param body body createRoleRequest true "Request body" // @Success 200 {object} createRoleResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -51,7 +52,8 @@ func (a *API) serveCreateRole(w http.ResponseWriter, r *http.Request) { }, ) - roleID, err := a.service.CreateRole(ctx, types.CreateRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + roleID, err := svc.CreateRole(ctx, types.CreateRoleRequest{ Name: httpReq.Name, Permissions: parsePermissionGroupsToStrings(httpReq.Permissions), }) @@ -69,6 +71,7 @@ func (a *API) serveCreateRole(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_add_users_to_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Param body body addUsersToRoleRequest true "Request body" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -102,7 +105,8 @@ func (a *API) serveAddUsersToRole(w http.ResponseWriter, r *http.Request) { }, ) - if err := a.service.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.AddUsersToRole(ctx, types.AddUsersToRoleRequest{ RoleID: roleID, Usernames: httpReq.Usernames, }); err != nil { @@ -119,6 +123,7 @@ func (a *API) serveAddUsersToRole(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_delete_users_from_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Param body body deleteUsersFromRoleRequest true "Request body" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -152,7 +157,8 @@ func (a *API) serveDeleteUsersFromRole(w http.ResponseWriter, r *http.Request) { }, ) - if err := a.service.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.DeleteUsersFromRole(ctx, types.DeleteUsersFromRoleRequest{ RoleID: roleID, Usernames: httpReq.Usernames, }); err != nil { @@ -168,6 +174,7 @@ func (a *API) serveDeleteUsersFromRole(w http.ResponseWriter, r *http.Request) { // @Router /admin/v1/roles [get] // @ID admin_v1_get_roles // @Tags admin_v1 +// @Param env query string false "Environment" // @Success 200 {object} getRolesResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer @@ -177,7 +184,8 @@ func (a *API) serveGetRoles(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - resp, err := a.service.GetRoles(ctx) + svc := a.services[types.GetEnvKey(ctx)] + resp, err := svc.GetRoles(ctx) if err != nil { httputil.ProcessError(wr, err) return @@ -195,6 +203,7 @@ func (a *API) serveGetRoles(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_get_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Success 200 {object} getRoleResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer @@ -217,7 +226,8 @@ func (a *API) serveGetRole(w http.ResponseWriter, r *http.Request) { }, ) - roleInfo, err := a.service.GetRole(ctx, types.GetRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + roleInfo, err := svc.GetRole(ctx, types.GetRoleRequest{ RoleID: roleID, }) if err != nil { @@ -236,6 +246,7 @@ func (a *API) serveGetRole(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_update_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Param body body updateRoleRequest true "Request body" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -276,7 +287,8 @@ func (a *API) serveUpdateRole(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(spanAttributes...) - if err := a.service.UpdateRole(ctx, types.UpdateRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.UpdateRole(ctx, types.UpdateRoleRequest{ RoleID: roleID, Name: httpReq.Name, Permissions: parsePermissionGroupsToStrings(httpReq.Permissions), @@ -294,6 +306,7 @@ func (a *API) serveUpdateRole(w http.ResponseWriter, r *http.Request) { // @ID admin_v1_delete_role // @Tags admin_v1 // @Param id path int32 true "Role ID" +// @Param env query string false "Environment" // @Param body body deleteRoleRequest false "Request body" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -330,7 +343,8 @@ func (a *API) serveDeleteRole(w http.ResponseWriter, r *http.Request) { } span.SetAttributes(spanAttributes...) - if err := a.service.DeleteRole(ctx, types.DeleteRoleRequest{ + svc := a.services[types.GetEnvKey(ctx)] + if err := svc.DeleteRole(ctx, types.DeleteRoleRequest{ RoleID: roleID, ReplacementRoleID: httpReq.ReplacementRoleID, }); err != nil { diff --git a/internal/api/admin/v1/http/roles_test.go b/internal/api/admin/v1/http/roles_test.go index 9876166..c4281f8 100644 --- a/internal/api/admin/v1/http/roles_test.go +++ b/internal/api/admin/v1/http/roles_test.go @@ -11,6 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + admin_service "github.com/ozontech/seq-ui/internal/pkg/service/admin" mock "github.com/ozontech/seq-ui/internal/pkg/service/admin/mock" ) @@ -32,7 +33,7 @@ func setupTestAPI(t *testing.T) (*API, *mock.MockService) { }). Times(1) - return New(mockedSvc), mockedSvc + return New(map[string]admin_service.Service{"": mockedSvc}), mockedSvc } func withRoleID(h http.HandlerFunc, id string) http.HandlerFunc { diff --git a/internal/api/errorgroups/v1/errorgroups.go b/internal/api/errorgroups/v1/errorgroups.go index dd3f341..97e0d7a 100644 --- a/internal/api/errorgroups/v1/errorgroups.go +++ b/internal/api/errorgroups/v1/errorgroups.go @@ -13,10 +13,10 @@ type ErrorGroups struct { httpAPI *http_api.API } -func New(svc errorgroups.Service) *ErrorGroups { +func New(services map[string]errorgroups.Service) *ErrorGroups { return &ErrorGroups{ - grpcAPI: grpc_api.New(svc), - httpAPI: http_api.New(svc), + grpcAPI: grpc_api.New(services), + httpAPI: http_api.New(services), } } diff --git a/internal/api/errorgroups/v1/grpc/api.go b/internal/api/errorgroups/v1/grpc/api.go index 62071e5..fb807ac 100644 --- a/internal/api/errorgroups/v1/grpc/api.go +++ b/internal/api/errorgroups/v1/grpc/api.go @@ -1,6 +1,8 @@ package grpc import ( + "context" + "google.golang.org/protobuf/types/known/durationpb" "github.com/ozontech/seq-ui/internal/app/types" @@ -11,13 +13,21 @@ import ( type API struct { api.UnimplementedErrorGroupsServiceServer - service errorgroups.Service + services map[string]errorgroups.Service } -func New(svc errorgroups.Service) *API { +func New(services map[string]errorgroups.Service) *API { return &API{ - service: svc, + services: services, + } +} + +func (a *API) serviceByEnv(ctx context.Context, reqEnv *string) (errorgroups.Service, *string) { + env := types.GetEnvKey(ctx) + if (reqEnv == nil || *reqEnv == "") && env != "" { + reqEnv = &env } + return a.services[env], reqEnv } type trReq interface { diff --git a/internal/api/errorgroups/v1/grpc/diff_by_releases.go b/internal/api/errorgroups/v1/grpc/diff_by_releases.go index f6e7fdd..7b68c33 100644 --- a/internal/api/errorgroups/v1/grpc/diff_by_releases.go +++ b/internal/api/errorgroups/v1/grpc/diff_by_releases.go @@ -16,6 +16,8 @@ func (a *API) DiffByReleases(ctx context.Context, req *errorgroups.DiffByRelease ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_diff_by_releases") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, {Key: "releases", Value: attribute.StringSliceValue(req.Releases)}, @@ -24,18 +26,18 @@ func (a *API) DiffByReleases(ctx context.Context, req *errorgroups.DiffByRelease {Key: "order", Value: attribute.StringValue(req.Order.String())}, {Key: "with_total", Value: attribute.BoolValue(req.WithTotal)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Source != nil { attributes = append(attributes, attribute.KeyValue{Key: "source", Value: attribute.StringValue(*req.Source)}) } span.SetAttributes(attributes...) - groups, total, err := a.service.DiffByReleases(ctx, types.DiffByReleasesRequest{ + groups, total, err := svc.DiffByReleases(ctx, types.DiffByReleasesRequest{ Service: req.Service, Releases: req.Releases, - Env: req.Env, + Env: reqEnv, Source: req.Source, Limit: req.Limit, Offset: req.Offset, diff --git a/internal/api/errorgroups/v1/grpc/diff_by_releases_test.go b/internal/api/errorgroups/v1/grpc/diff_by_releases_test.go index d0cef71..ddd6948 100644 --- a/internal/api/errorgroups/v1/grpc/diff_by_releases_test.go +++ b/internal/api/errorgroups/v1/grpc/diff_by_releases_test.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -145,7 +146,7 @@ func TestDiffByReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_details.go b/internal/api/errorgroups/v1/grpc/get_details.go index a923fa8..091d50c 100644 --- a/internal/api/errorgroups/v1/grpc/get_details.go +++ b/internal/api/errorgroups/v1/grpc/get_details.go @@ -20,14 +20,16 @@ func (a *API) GetDetails(ctx context.Context, req *errorgroups.GetDetailsRequest ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_details") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "group_hash", Value: attribute.StringValue(strconv.FormatUint(req.GroupHash, 10))}, } if req.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*req.Service)}) } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*req.Release)}) @@ -44,7 +46,7 @@ func (a *API) GetDetails(ctx context.Context, req *errorgroups.GetDetailsRequest request := types.GetErrorGroupDetailsRequest{ Service: req.Service, GroupHash: req.GroupHash, - Env: req.Env, + Env: reqEnv, Source: req.Source, Release: req.Release, } @@ -55,7 +57,7 @@ func (a *API) GetDetails(ctx context.Context, req *errorgroups.GetDetailsRequest } } - details, err := a.service.GetDetails(ctx, request) + details, err := svc.GetDetails(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_details_test.go b/internal/api/errorgroups/v1/grpc/get_details_test.go index 2a0d007..21c7191 100644 --- a/internal/api/errorgroups/v1/grpc/get_details_test.go +++ b/internal/api/errorgroups/v1/grpc/get_details_test.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -261,7 +262,7 @@ func TestGetDetails(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_groups.go b/internal/api/errorgroups/v1/grpc/get_groups.go index 0a589ea..52baad5 100644 --- a/internal/api/errorgroups/v1/grpc/get_groups.go +++ b/internal/api/errorgroups/v1/grpc/get_groups.go @@ -17,6 +17,8 @@ func (a *API) GetGroups(ctx context.Context, req *errorgroups.GetGroupsRequest) ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_groups") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, {Key: "limit", Value: attribute.IntValue(int(req.Limit))}, @@ -24,8 +26,8 @@ func (a *API) GetGroups(ctx context.Context, req *errorgroups.GetGroupsRequest) {Key: "order", Value: attribute.StringValue(string(req.Order))}, {Key: "with_total", Value: attribute.BoolValue(req.WithTotal)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*req.Release)}) @@ -48,7 +50,7 @@ func (a *API) GetGroups(ctx context.Context, req *errorgroups.GetGroupsRequest) request := types.GetErrorGroupsRequest{ Service: req.Service, - Env: req.Env, + Env: reqEnv, Source: req.Source, Release: req.Release, TimeRange: parseTimeRange(req), @@ -70,9 +72,9 @@ func (a *API) GetGroups(ctx context.Context, req *errorgroups.GetGroupsRequest) err error ) if req.Filter != nil && req.Filter.IsNew { - groups, total, err = a.service.GetNewErrorGroups(ctx, request) + groups, total, err = svc.GetNewErrorGroups(ctx, request) } else { - groups, total, err = a.service.GetErrorGroups(ctx, request) + groups, total, err = svc.GetErrorGroups(ctx, request) } if err != nil { diff --git a/internal/api/errorgroups/v1/grpc/get_groups_test.go b/internal/api/errorgroups/v1/grpc/get_groups_test.go index 2ec2993..9083eb5 100644 --- a/internal/api/errorgroups/v1/grpc/get_groups_test.go +++ b/internal/api/errorgroups/v1/grpc/get_groups_test.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -445,7 +446,7 @@ func TestGetGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { if tt.req.Filter != nil && tt.req.Filter.IsNew { diff --git a/internal/api/errorgroups/v1/grpc/get_hist.go b/internal/api/errorgroups/v1/grpc/get_hist.go index 6672e12..eaf1649 100644 --- a/internal/api/errorgroups/v1/grpc/get_hist.go +++ b/internal/api/errorgroups/v1/grpc/get_hist.go @@ -18,6 +18,8 @@ func (a *API) GetHist(ctx context.Context, req *errorgroups.GetHistRequest) (*er ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_hist") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{} if req.GroupHash != nil { attributes = append(attributes, attribute.KeyValue{ @@ -27,8 +29,8 @@ func (a *API) GetHist(ctx context.Context, req *errorgroups.GetHistRequest) (*er if req.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*req.Service)}) } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*req.Release)}) @@ -52,7 +54,7 @@ func (a *API) GetHist(ctx context.Context, req *errorgroups.GetHistRequest) (*er request := types.GetErrorHistRequest{ Service: req.Service, GroupHash: req.GroupHash, - Env: req.Env, + Env: reqEnv, Source: req.Source, Release: req.Release, TimeRange: parseTimeRange(req), @@ -64,7 +66,7 @@ func (a *API) GetHist(ctx context.Context, req *errorgroups.GetHistRequest) (*er } } - hist, err := a.service.GetHist(ctx, request) + hist, err := svc.GetHist(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_hist_test.go b/internal/api/errorgroups/v1/grpc/get_hist_test.go index db06525..04c3c72 100644 --- a/internal/api/errorgroups/v1/grpc/get_hist_test.go +++ b/internal/api/errorgroups/v1/grpc/get_hist_test.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -231,7 +232,7 @@ func TestGetHist(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_releases.go b/internal/api/errorgroups/v1/grpc/get_releases.go index 1a5ff8b..7f27655 100644 --- a/internal/api/errorgroups/v1/grpc/get_releases.go +++ b/internal/api/errorgroups/v1/grpc/get_releases.go @@ -15,19 +15,21 @@ func (a *API) GetReleases(ctx context.Context, req *errorgroups.GetReleasesReque ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_releases") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } span.SetAttributes(attributes...) request := types.GetReleasesRequest{ Service: req.Service, - Env: req.Env, + Env: reqEnv, } - releases, err := a.service.GetReleases(ctx, request) + releases, err := svc.GetReleases(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_releases_test.go b/internal/api/errorgroups/v1/grpc/get_releases_test.go index 670bad8..757e993 100644 --- a/internal/api/errorgroups/v1/grpc/get_releases_test.go +++ b/internal/api/errorgroups/v1/grpc/get_releases_test.go @@ -9,6 +9,7 @@ import ( "go.uber.org/mock/gomock" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -76,7 +77,7 @@ func TestGetReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_service_filters.go b/internal/api/errorgroups/v1/grpc/get_service_filters.go index 4b669a7..102a0b2 100644 --- a/internal/api/errorgroups/v1/grpc/get_service_filters.go +++ b/internal/api/errorgroups/v1/grpc/get_service_filters.go @@ -15,19 +15,21 @@ func (a *API) GetServiceFilters(ctx context.Context, req *errorgroups.GetService ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_service_filters") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(req.Service)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } span.SetAttributes(attributes...) request := types.GetServiceFiltersRequest{ Service: req.Service, - Env: req.Env, + Env: reqEnv, } - filters, err := a.service.GetServiceFilters(ctx, request) + filters, err := svc.GetServiceFilters(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_service_filters_test.go b/internal/api/errorgroups/v1/grpc/get_service_filters_test.go index d722e1e..73082ab 100644 --- a/internal/api/errorgroups/v1/grpc/get_service_filters_test.go +++ b/internal/api/errorgroups/v1/grpc/get_service_filters_test.go @@ -9,6 +9,7 @@ import ( "go.uber.org/mock/gomock" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -83,7 +84,7 @@ func TestGetServiceFilters(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_services.go b/internal/api/errorgroups/v1/grpc/get_services.go index 69aa0e4..215eb2e 100644 --- a/internal/api/errorgroups/v1/grpc/get_services.go +++ b/internal/api/errorgroups/v1/grpc/get_services.go @@ -15,21 +15,23 @@ func (a *API) GetServices(ctx context.Context, req *errorgroups.GetServicesReque ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_services") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "query", Value: attribute.StringValue(req.Query)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } span.SetAttributes(attributes...) request := types.GetServicesRequest{ Query: req.Query, - Env: req.Env, + Env: reqEnv, Limit: req.Limit, Offset: req.Offset, } - services, err := a.service.GetServices(ctx, request) + services, err := svc.GetServices(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_services_test.go b/internal/api/errorgroups/v1/grpc/get_services_test.go index 2b2b9e5..4ff3ba5 100644 --- a/internal/api/errorgroups/v1/grpc/get_services_test.go +++ b/internal/api/errorgroups/v1/grpc/get_services_test.go @@ -9,6 +9,7 @@ import ( "go.uber.org/mock/gomock" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -81,7 +82,7 @@ func TestGetServices(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/grpc/get_top_groups.go b/internal/api/errorgroups/v1/grpc/get_top_groups.go index 649150f..475122b 100644 --- a/internal/api/errorgroups/v1/grpc/get_top_groups.go +++ b/internal/api/errorgroups/v1/grpc/get_top_groups.go @@ -16,13 +16,15 @@ func (a *API) GetTopGroups(ctx context.Context, req *errorgroups.GetTopGroupsReq ctx, span := tracing.StartSpan(ctx, "errorgroups_v1_get_top_groups") defer span.End() + svc, reqEnv := a.serviceByEnv(ctx, req.Env) + attributes := []attribute.KeyValue{ {Key: "limit", Value: attribute.IntValue(int(req.Limit))}, {Key: "offset", Value: attribute.IntValue(int(req.Offset))}, {Key: "with_total", Value: attribute.BoolValue(req.WithTotal)}, } - if req.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*req.Env)}) + if reqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*reqEnv)}) } if req.Source != nil { attributes = append(attributes, attribute.KeyValue{Key: "source", Value: attribute.StringValue(*req.Source)}) @@ -37,7 +39,7 @@ func (a *API) GetTopGroups(ctx context.Context, req *errorgroups.GetTopGroupsReq span.SetAttributes(attributes...) request := types.GetTopErrorGroupsRequest{ - Env: req.Env, + Env: reqEnv, Source: req.Source, TimeRange: parseTimeRange(req), Limit: req.Limit, @@ -45,7 +47,7 @@ func (a *API) GetTopGroups(ctx context.Context, req *errorgroups.GetTopGroupsReq WithTotal: req.WithTotal, } - groups, total, err := a.service.GetTopErrorGroups(ctx, request) + groups, total, err := svc.GetTopErrorGroups(ctx, request) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/errorgroups/v1/grpc/get_top_groups_test.go b/internal/api/errorgroups/v1/grpc/get_top_groups_test.go index c276f1e..4297cdb 100644 --- a/internal/api/errorgroups/v1/grpc/get_top_groups_test.go +++ b/internal/api/errorgroups/v1/grpc/get_top_groups_test.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" errorgroups_v1 "github.com/ozontech/seq-ui/pkg/errorgroups/v1" ) @@ -239,7 +240,7 @@ func TestGetTopGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/api.go b/internal/api/errorgroups/v1/http/api.go index 71711d1..3ca9378 100644 --- a/internal/api/errorgroups/v1/http/api.go +++ b/internal/api/errorgroups/v1/http/api.go @@ -1,6 +1,7 @@ package http import ( + "context" "strconv" "time" @@ -11,15 +12,23 @@ import ( ) type API struct { - service errorgroups.Service + services map[string]errorgroups.Service } -func New(svc errorgroups.Service) *API { +func New(services map[string]errorgroups.Service) *API { return &API{ - service: svc, + services: services, } } +func (a *API) serviceByEnv(ctx context.Context, reqEnv *string) (errorgroups.Service, *string) { + env := types.GetEnvKey(ctx) + if (reqEnv == nil || *reqEnv == "") && env != "" { + reqEnv = &env + } + return a.services[env], reqEnv +} + func (a *API) Router() chi.Router { mux := chi.NewMux() diff --git a/internal/api/errorgroups/v1/http/diff_by_releases.go b/internal/api/errorgroups/v1/http/diff_by_releases.go index c6e2299..d7f7e0b 100644 --- a/internal/api/errorgroups/v1/http/diff_by_releases.go +++ b/internal/api/errorgroups/v1/http/diff_by_releases.go @@ -19,6 +19,7 @@ import ( // @Router /errorgroups/v1/diff_by_releases [post] // @ID errorgroups_v1_diff_by_releases // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body diffByReleasesRequest true "Request body" // @Success 200 {object} diffByReleasesResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -35,6 +36,8 @@ func (a *API) serveDiffByReleases(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, {Key: "releases", Value: attribute.StringSliceValue(httpReq.Releases)}, @@ -43,18 +46,18 @@ func (a *API) serveDiffByReleases(w http.ResponseWriter, r *http.Request) { {Key: "order", Value: attribute.StringValue(string(httpReq.Order))}, {Key: "with_total", Value: attribute.BoolValue(httpReq.WithTotal)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Source != nil { attributes = append(attributes, attribute.KeyValue{Key: "source", Value: attribute.StringValue(*httpReq.Source)}) } span.SetAttributes(attributes...) - groups, total, err := a.service.DiffByReleases(ctx, types.DiffByReleasesRequest{ + groups, total, err := svc.DiffByReleases(ctx, types.DiffByReleasesRequest{ Service: httpReq.Service, Releases: httpReq.Releases, - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, Limit: httpReq.Limit, Offset: httpReq.Offset, diff --git a/internal/api/errorgroups/v1/http/diff_by_releases_test.go b/internal/api/errorgroups/v1/http/diff_by_releases_test.go index 8e27968..1e09d1e 100644 --- a/internal/api/errorgroups/v1/http/diff_by_releases_test.go +++ b/internal/api/errorgroups/v1/http/diff_by_releases_test.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -144,7 +145,7 @@ func TestServeDiffByReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_details.go b/internal/api/errorgroups/v1/http/get_details.go index 39d0156..3b8759a 100644 --- a/internal/api/errorgroups/v1/http/get_details.go +++ b/internal/api/errorgroups/v1/http/get_details.go @@ -20,6 +20,7 @@ import ( // @Router /errorgroups/v1/details [post] // @ID errorgroups_v1_get_details // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getDetailsRequest true "Request body" // @Success 200 {object} getDetailsResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -36,6 +37,8 @@ func (a *API) serveGetDetails(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + parsedGroupHash, err := parseGroupHash(&httpReq.GroupHash) if err != nil { wr.Error(fmt.Errorf("failed to parse group_hash: %w", err), http.StatusBadRequest) @@ -45,8 +48,8 @@ func (a *API) serveGetDetails(w http.ResponseWriter, r *http.Request) { attributes := []attribute.KeyValue{ {Key: "group_hash", Value: attribute.StringValue(httpReq.GroupHash)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*httpReq.Service)}) @@ -65,7 +68,7 @@ func (a *API) serveGetDetails(w http.ResponseWriter, r *http.Request) { req := types.GetErrorGroupDetailsRequest{ GroupHash: *parsedGroupHash, - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, Service: httpReq.Service, Release: httpReq.Release, @@ -77,7 +80,7 @@ func (a *API) serveGetDetails(w http.ResponseWriter, r *http.Request) { } } - details, err := a.service.GetDetails(ctx, req) + details, err := svc.GetDetails(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_details_test.go b/internal/api/errorgroups/v1/http/get_details_test.go index 88fb479..a954774 100644 --- a/internal/api/errorgroups/v1/http/get_details_test.go +++ b/internal/api/errorgroups/v1/http/get_details_test.go @@ -11,6 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -260,7 +261,7 @@ func TestServeGetDetails(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_groups.go b/internal/api/errorgroups/v1/http/get_groups.go index 41c2ddb..15580d6 100644 --- a/internal/api/errorgroups/v1/http/get_groups.go +++ b/internal/api/errorgroups/v1/http/get_groups.go @@ -19,6 +19,7 @@ import ( // @Router /errorgroups/v1/groups [post] // @ID errorgroups_v1_get_groups // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getGroupsRequest true "Request body" // @Success 200 {object} getGroupsResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -35,6 +36,8 @@ func (a *API) serveGetGroups(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, {Key: "limit", Value: attribute.IntValue(int(httpReq.Limit))}, @@ -42,8 +45,8 @@ func (a *API) serveGetGroups(w http.ResponseWriter, r *http.Request) { {Key: "order", Value: attribute.StringValue(string(httpReq.Order))}, {Key: "with_total", Value: attribute.BoolValue(httpReq.WithTotal)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*httpReq.Release)}) @@ -72,7 +75,7 @@ func (a *API) serveGetGroups(w http.ResponseWriter, r *http.Request) { req := types.GetErrorGroupsRequest{ Service: httpReq.Service, - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, Release: httpReq.Release, TimeRange: tr, @@ -93,9 +96,9 @@ func (a *API) serveGetGroups(w http.ResponseWriter, r *http.Request) { total uint64 ) if httpReq.Filter != nil && httpReq.Filter.IsNew { - groups, total, err = a.service.GetNewErrorGroups(ctx, req) + groups, total, err = svc.GetNewErrorGroups(ctx, req) } else { - groups, total, err = a.service.GetErrorGroups(ctx, req) + groups, total, err = svc.GetErrorGroups(ctx, req) } if err != nil { diff --git a/internal/api/errorgroups/v1/http/get_groups_test.go b/internal/api/errorgroups/v1/http/get_groups_test.go index 2c532b1..56ef5a5 100644 --- a/internal/api/errorgroups/v1/http/get_groups_test.go +++ b/internal/api/errorgroups/v1/http/get_groups_test.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -439,7 +440,7 @@ func TestServeGetGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { if tt.req.Filter != nil && tt.req.Filter.IsNew { diff --git a/internal/api/errorgroups/v1/http/get_hist.go b/internal/api/errorgroups/v1/http/get_hist.go index 47919b9..b3752ca 100644 --- a/internal/api/errorgroups/v1/http/get_hist.go +++ b/internal/api/errorgroups/v1/http/get_hist.go @@ -18,6 +18,7 @@ import ( // @Router /errorgroups/v1/hist [post] // @ID errorgroups_v1_get_hist // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getHistRequest true "Request body" // @Success 200 {object} getHistResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -34,6 +35,8 @@ func (a *API) serveGetHist(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{} if httpReq.GroupHash != nil { attributes = append(attributes, attribute.KeyValue{Key: "group_hash", Value: attribute.StringValue(*httpReq.GroupHash)}) @@ -41,8 +44,8 @@ func (a *API) serveGetHist(w http.ResponseWriter, r *http.Request) { if httpReq.Service != nil { attributes = append(attributes, attribute.KeyValue{Key: "service", Value: attribute.StringValue(*httpReq.Service)}) } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Release != nil { attributes = append(attributes, attribute.KeyValue{Key: "release", Value: attribute.StringValue(*httpReq.Release)}) @@ -78,7 +81,7 @@ func (a *API) serveGetHist(w http.ResponseWriter, r *http.Request) { req := types.GetErrorHistRequest{ Service: httpReq.Service, GroupHash: parsedGroupHash, - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, Release: httpReq.Release, TimeRange: tr, @@ -90,7 +93,7 @@ func (a *API) serveGetHist(w http.ResponseWriter, r *http.Request) { } } - hist, err := a.service.GetHist(ctx, req) + hist, err := svc.GetHist(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_hist_test.go b/internal/api/errorgroups/v1/http/get_hist_test.go index 249fa40..7692410 100644 --- a/internal/api/errorgroups/v1/http/get_hist_test.go +++ b/internal/api/errorgroups/v1/http/get_hist_test.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -253,7 +254,7 @@ func TestServeGetHist(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_releases.go b/internal/api/errorgroups/v1/http/get_releases.go index 2a49974..2a79f5a 100644 --- a/internal/api/errorgroups/v1/http/get_releases.go +++ b/internal/api/errorgroups/v1/http/get_releases.go @@ -17,6 +17,7 @@ import ( // @Router /errorgroups/v1/releases [post] // @ID errorgroups_v1_get_releases // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getReleasesRequest true "Request body" // @Success 200 {object} getReleasesResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -33,19 +34,21 @@ func (a *API) serveGetReleases(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } span.SetAttributes(attributes...) req := types.GetReleasesRequest{ Service: httpReq.Service, - Env: httpReq.Env, + Env: httpReqEnv, } - releases, err := a.service.GetReleases(ctx, req) + releases, err := svc.GetReleases(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_releases_test.go b/internal/api/errorgroups/v1/http/get_releases_test.go index a603b92..4401946 100644 --- a/internal/api/errorgroups/v1/http/get_releases_test.go +++ b/internal/api/errorgroups/v1/http/get_releases_test.go @@ -9,6 +9,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -76,7 +77,7 @@ func TestServeGetReleases(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_service_filters.go b/internal/api/errorgroups/v1/http/get_service_filters.go index 233d367..3b27e52 100644 --- a/internal/api/errorgroups/v1/http/get_service_filters.go +++ b/internal/api/errorgroups/v1/http/get_service_filters.go @@ -17,6 +17,7 @@ import ( // @Router /errorgroups/v1/service_filters [post] // @ID errorgroups_v1_get_service_filters // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getServiceFiltersRequest true "Request body" // @Success 200 {object} getServiceFiltersResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -33,19 +34,21 @@ func (a *API) serveGetServiceFilters(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "service", Value: attribute.StringValue(httpReq.Service)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } span.SetAttributes(attributes...) req := types.GetServiceFiltersRequest{ Service: httpReq.Service, - Env: httpReq.Env, + Env: httpReqEnv, } - filters, err := a.service.GetServiceFilters(ctx, req) + filters, err := svc.GetServiceFilters(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_service_filters_test.go b/internal/api/errorgroups/v1/http/get_service_filters_test.go index ecbfdec..80909cf 100644 --- a/internal/api/errorgroups/v1/http/get_service_filters_test.go +++ b/internal/api/errorgroups/v1/http/get_service_filters_test.go @@ -9,6 +9,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -82,7 +83,7 @@ func TestServeGetServiceFilters(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_services.go b/internal/api/errorgroups/v1/http/get_services.go index e02eea8..f630b1f 100644 --- a/internal/api/errorgroups/v1/http/get_services.go +++ b/internal/api/errorgroups/v1/http/get_services.go @@ -17,6 +17,7 @@ import ( // @Router /errorgroups/v1/services [post] // @ID errorgroups_v1_get_services // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getServicesRequest true "Request body" // @Success 200 {object} getServicesResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -33,21 +34,23 @@ func (a *API) serveGetServices(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "query", Value: attribute.StringValue(httpReq.Query)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } span.SetAttributes(attributes...) req := types.GetServicesRequest{ Query: httpReq.Query, - Env: httpReq.Env, + Env: httpReqEnv, Limit: httpReq.Limit, Offset: httpReq.Offset, } - services, err := a.service.GetServices(ctx, req) + services, err := svc.GetServices(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_services_test.go b/internal/api/errorgroups/v1/http/get_services_test.go index bd63165..f68abc0 100644 --- a/internal/api/errorgroups/v1/http/get_services_test.go +++ b/internal/api/errorgroups/v1/http/get_services_test.go @@ -9,6 +9,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -79,7 +80,7 @@ func TestServeGetServices(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/errorgroups/v1/http/get_top_groups.go b/internal/api/errorgroups/v1/http/get_top_groups.go index c04d65d..ea009e6 100644 --- a/internal/api/errorgroups/v1/http/get_top_groups.go +++ b/internal/api/errorgroups/v1/http/get_top_groups.go @@ -18,6 +18,7 @@ import ( // @Router /errorgroups/v1/top_groups [post] // @ID errorgroups_v1_get_top_groups // @Tags errorgroups_v1 +// @Param env query string false "Environment" // @Param body body getTopGroupsRequest true "Request body" // @Success 200 {object} getTopGroupsResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" @@ -34,13 +35,15 @@ func (a *API) serveGetTopGroups(w http.ResponseWriter, r *http.Request) { return } + svc, httpReqEnv := a.serviceByEnv(ctx, httpReq.Env) + attributes := []attribute.KeyValue{ {Key: "limit", Value: attribute.IntValue(int(httpReq.Limit))}, {Key: "offset", Value: attribute.IntValue(int(httpReq.Offset))}, {Key: "with_total", Value: attribute.BoolValue(httpReq.WithTotal)}, } - if httpReq.Env != nil { - attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReq.Env)}) + if httpReqEnv != nil { + attributes = append(attributes, attribute.KeyValue{Key: "env", Value: attribute.StringValue(*httpReqEnv)}) } if httpReq.Source != nil { attributes = append(attributes, attribute.KeyValue{Key: "source", Value: attribute.StringValue(*httpReq.Source)}) @@ -61,7 +64,7 @@ func (a *API) serveGetTopGroups(w http.ResponseWriter, r *http.Request) { } req := types.GetTopErrorGroupsRequest{ - Env: httpReq.Env, + Env: httpReqEnv, Source: httpReq.Source, TimeRange: tr, Limit: httpReq.Limit, @@ -69,7 +72,7 @@ func (a *API) serveGetTopGroups(w http.ResponseWriter, r *http.Request) { WithTotal: httpReq.WithTotal, } - groups, total, err := a.service.GetTopErrorGroups(ctx, req) + groups, total, err := svc.GetTopErrorGroups(ctx, req) if err != nil { httputil.ProcessError(wr, err) return diff --git a/internal/api/errorgroups/v1/http/get_top_groups_test.go b/internal/api/errorgroups/v1/http/get_top_groups_test.go index 7737259..546b6cb 100644 --- a/internal/api/errorgroups/v1/http/get_top_groups_test.go +++ b/internal/api/errorgroups/v1/http/get_top_groups_test.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups" svc_mock "github.com/ozontech/seq-ui/internal/pkg/service/errorgroups/mock" ) @@ -236,7 +237,7 @@ func TestServeGetTopGroups(t *testing.T) { ctrl := gomock.NewController(t) mockedSvc := svc_mock.NewMockService(ctrl) - api := New(mockedSvc) + api := New(map[string]errorgroups.Service{"": mockedSvc}) if ma := tt.mockArgs; ma != nil { mockedSvc.EXPECT(). diff --git a/internal/api/seqapi/v1/grpc/aggregation.go b/internal/api/seqapi/v1/grpc/aggregation.go index e8fd760..53c8101 100644 --- a/internal/api/seqapi/v1/grpc/aggregation.go +++ b/internal/api/seqapi/v1/grpc/aggregation.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/seqapi/v1/aggregation_ts" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -19,7 +20,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ defer span.End() aggregations, _ := json.Marshal(req.Aggregations) - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) attributes := []attribute.KeyValue{ { @@ -59,7 +60,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ return nil, status.Error(codes.InvalidArgument, err.Error()) } - if err := api_error.CheckAggregationsCount(len(req.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(req.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -69,7 +70,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ continue } if err := api_error.CheckAggregationTsInterval(*agg.Interval, fromRaw, toRaw, - params.options.MaxBucketsPerAggregationTs, + params.options.Limits.MaxBucketsPerAggregationTs, ); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } diff --git a/internal/api/seqapi/v1/grpc/aggregation_test.go b/internal/api/seqapi/v1/grpc/aggregation_test.go index 785a824..f4c1a0e 100644 --- a/internal/api/seqapi/v1/grpc/aggregation_test.go +++ b/internal/api/seqapi/v1/grpc/aggregation_test.go @@ -12,7 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -50,8 +50,10 @@ func TestGetAggregation(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -65,8 +67,10 @@ func TestGetAggregation(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 2, + }, }, }, apiErr: true, @@ -82,8 +86,10 @@ func TestGetAggregation(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 1, + }, }, }, clientErr: errors.New("client error"), @@ -178,8 +184,10 @@ func TestGetAggregationWithNormalization(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -221,8 +229,10 @@ func TestGetAggregationWithNormalization(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index da9a784..080dc7d 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -1,16 +1,14 @@ package grpc import ( - "context" "fmt" "sort" "strconv" "time" "go.uber.org/zap" - "google.golang.org/grpc/metadata" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" @@ -21,104 +19,90 @@ import ( ) type apiParams struct { - client seqdb.Client - options *config.SeqAPIOptions - fieldsCache *fieldsCache - masker *mask.Masker - pinnedFields []*seqapi.Field - systemFields []*seqapi.Field + client seqdb.Client + options *config.SeqAPIOptions + masker *mask.Masker + pinnedFields []*seqapi.Field + systemFields []*seqapi.Field + eventsCache cache.Cache + logsLifespanCache cache.Cache + fieldsCache *fieldsCache + eventsCacheKeyPrefix string + fieldsCacheKey string + logsLifespanCacheKey string } type API struct { seqapi.UnimplementedSeqAPIServiceServer - config config.SeqAPI - params apiParams - paramsByEnv map[string]apiParams - inmemWithRedisCache cache.Cache - redisCache cache.Cache - nowFn func() time.Time - asyncSearches asyncsearches.Service - envsResponse *seqapi.GetEnvsResponse + config *config.SeqAPI + paramsByEnv map[string]apiParams + nowFn func() time.Time + asyncSearches map[string]asyncsearches.Service + envsResponse *seqapi.GetEnvsResponse } -func New( - cfg config.SeqAPI, - seqDBСlients map[string]seqdb.Client, - inmemWithRedisCache cache.Cache, - redisCache cache.Cache, - asyncSearches asyncsearches.Service, -) *API { - var globalfCache *fieldsCache - if cfg.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.FieldsCacheTTL) - } - - globalMasker, err := mask.New(cfg.Masking) - if err != nil { - logger.Fatal("failed to init masking", zap.Error(err)) - } - - globalPinnedFields := parseFields(cfg.PinnedFields) - globalSystemFields := parseFields(cfg.SystemFields) - - var params apiParams - var paramsByEnv map[string]apiParams - +func New(cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, asyncSearches map[string]asyncsearches.Service) *API { + paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { - paramsByEnv = make(map[string]apiParams) for envName, envConfig := range cfg.Envs { - client := seqDBСlients[envConfig.SeqDB] - options := envConfig.Options + paramsByEnv[envName] = newAPIParams(envName, seqDBСlients[envConfig.SeqDBID], envConfig.Options) + } + } else { + var client seqdb.Client + for _, c := range seqDBСlients { + client = c + } + paramsByEnv[""] = newAPIParams("", client, &cfg.Options) + } - var envfCache *fieldsCache - if options.FieldsCacheTTL > 0 { - envfCache = newFieldsCache(options.FieldsCacheTTL) - } + return &API{ + config: cfg, + paramsByEnv: paramsByEnv, + nowFn: time.Now, + asyncSearches: asyncSearches, + envsResponse: parseEnvs(cfg), + } +} - var envMasker, err = mask.New(options.Masking) - if err != nil { - logger.Fatal("failed to init env masking", zap.Error(err)) - } +func newAPIParams(env string, client seqdb.Client, options *config.SeqAPIOptions) apiParams { + masker, err := mask.New(options.Masking) + if err != nil { + logger.Fatal("failed to init masking", zap.String("env", env), zap.Error(err)) + } - envPinnedFields := parseFields(options.PinnedFields) - envSystemFields := parseFields(options.SystemFields) + var eventsCache, logsLifespanCache cache.Cache + if options.Caches.Events != nil { + eventsCache, _ = cache.ByID(options.Caches.Events.ID) + } + if options.Caches.LogsLifespan != nil { + logsLifespanCache, _ = cache.ByID(options.Caches.LogsLifespan.ID) + } - paramsByEnv[envName] = apiParams{ - client: client, - options: options, - fieldsCache: envfCache, - masker: envMasker, - pinnedFields: envPinnedFields, - systemFields: envSystemFields, - } - } - } else { - client, exists := seqDBСlients[config.DefaultSeqDBClientID] - if !exists { - logger.Fatal("default client not found", - zap.String("clientID", config.DefaultSeqDBClientID)) + var fc *fieldsCache + if options.Caches.Fields != nil { + if c, ok := cache.ByID(options.Caches.Fields.ID); ok { + fc = newFieldsCache(c) } + } - params = apiParams{ - client: client, - options: cfg.SeqAPIOptions, - fieldsCache: globalfCache, - masker: globalMasker, - pinnedFields: globalPinnedFields, - systemFields: globalSystemFields, - } + cacheKeyPrefix := "seqapi_" + if env != "" { + cacheKeyPrefix += env + "_" } - return &API{ - config: cfg, - params: params, - paramsByEnv: paramsByEnv, - inmemWithRedisCache: inmemWithRedisCache, - redisCache: redisCache, - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + return apiParams{ + client: client, + options: options, + masker: masker, + pinnedFields: parseFields(options.PinnedFields), + systemFields: parseFields(options.SystemFields), + eventsCache: eventsCache, + logsLifespanCache: logsLifespanCache, + fieldsCache: fc, + eventsCacheKeyPrefix: cacheKeyPrefix + "event_", + fieldsCacheKey: cacheKeyPrefix + "fields", + logsLifespanCacheKey: cacheKeyPrefix + "logs_lifespan", } } @@ -133,7 +117,7 @@ func parseFields(fields []config.Field) []*seqapi.Field { return res } -func parseEnvs(cfg config.SeqAPI) *seqapi.GetEnvsResponse { +func parseEnvs(cfg *config.SeqAPI) *seqapi.GetEnvsResponse { var envs []*seqapi.GetEnvsResponse_Env if len(cfg.Envs) > 0 { // sort environment names to ensure deterministic output @@ -177,23 +161,14 @@ func parseEnvs(cfg config.SeqAPI) *seqapi.GetEnvsResponse { envs = append(envs, createEnvInfo(name, envConfig.Options)) } } else { - envs = []*seqapi.GetEnvsResponse_Env{createEnvInfo("", cfg.SeqAPIOptions)} + envs = []*seqapi.GetEnvsResponse_Env{createEnvInfo("", &cfg.Options)} } return &seqapi.GetEnvsResponse{Envs: envs} } -func (a *API) GetEnvFromContext(ctx context.Context) string { - md, _ := metadata.FromIncomingContext(ctx) - envValues := md.Get("env") - if len(envValues) == 0 { - return "" - } - return envValues[0] -} - func (a *API) GetParams(env string) (apiParams, error) { if len(a.config.Envs) == 0 { - return a.params, nil + return a.paramsByEnv[""], nil } if env == "" { @@ -208,26 +183,6 @@ func (a *API) GetParams(env string) (apiParams, error) { return params, nil } -type fieldsCache struct { - ttl time.Duration - - ts time.Time - fields []*seqapi.Field -} - -func newFieldsCache(ttl time.Duration) *fieldsCache { - return &fieldsCache{ttl: ttl} -} - -func (c *fieldsCache) getFields() ([]*seqapi.Field, bool, bool) { - return c.fields, !c.ts.IsZero(), time.Since(c.ts) < c.ttl -} - -func (c *fieldsCache) setFields(fields []*seqapi.Field) { - c.fields = fields - c.ts = time.Now() -} - func checkLimitOffset(limit, offset int32) error { if limit < 0 { return types.NewErrInvalidRequestField("'limit' must be non-negative") @@ -241,10 +196,10 @@ func checkLimitOffset(limit, offset int32) error { func createEnvInfo(envName string, opts *config.SeqAPIOptions) *seqapi.GetEnvsResponse_Env { return &seqapi.GetEnvsResponse_Env{ Env: envName, - MaxSearchLimit: uint32(opts.MaxSearchLimit), - MaxExportLimit: uint32(opts.MaxExportLimit), - MaxParallelExportRequests: uint32(opts.MaxParallelExportRequests), - MaxAggregationsPerRequest: uint32(opts.MaxAggregationsPerRequest), - SeqCliMaxSearchLimit: uint32(opts.SeqCLIMaxSearchLimit), + MaxSearchLimit: uint32(opts.Limits.MaxSearchLimit), + MaxExportLimit: uint32(opts.Limits.MaxExportLimit), + MaxParallelExportRequests: uint32(opts.Limits.MaxParallelExportRequests), + MaxAggregationsPerRequest: uint32(opts.Limits.MaxAggregationsPerRequest), + SeqCliMaxSearchLimit: uint32(opts.Limits.SeqCLIMaxSearchLimit), } } diff --git a/internal/api/seqapi/v1/grpc/cancel_async_search.go b/internal/api/seqapi/v1/grpc/cancel_async_search.go index 2506f36..00f2838 100644 --- a/internal/api/seqapi/v1/grpc/cancel_async_search.go +++ b/internal/api/seqapi/v1/grpc/cancel_async_search.go @@ -15,7 +15,8 @@ import ( ) func (a *API) CancelAsyncSearch(ctx context.Context, req *seqapi.CancelAsyncSearchRequest) (*seqapi.CancelAsyncSearchResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -33,7 +34,7 @@ func (a *API) CancelAsyncSearch(ctx context.Context, req *seqapi.CancelAsyncSear }, ) - resp, err := a.asyncSearches.CancelAsyncSearch(ctx, req) + resp, err := asyncSearches.CancelAsyncSearch(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/cluster_status.go b/internal/api/seqapi/v1/grpc/cluster_status.go index 0b09e6e..06f6b0e 100644 --- a/internal/api/seqapi/v1/grpc/cluster_status.go +++ b/internal/api/seqapi/v1/grpc/cluster_status.go @@ -6,6 +6,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -14,7 +15,7 @@ func (a *API) Status(ctx context.Context, req *seqapi.StatusRequest) (*seqapi.St ctx, span := tracing.StartSpan(ctx, "seqapi_v1_status") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/internal/api/seqapi/v1/grpc/delete_async_search.go b/internal/api/seqapi/v1/grpc/delete_async_search.go index 4c573ea..63f2fed 100644 --- a/internal/api/seqapi/v1/grpc/delete_async_search.go +++ b/internal/api/seqapi/v1/grpc/delete_async_search.go @@ -15,7 +15,8 @@ import ( ) func (a *API) DeleteAsyncSearch(ctx context.Context, req *seqapi.DeleteAsyncSearchRequest) (*seqapi.DeleteAsyncSearchResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -33,7 +34,7 @@ func (a *API) DeleteAsyncSearch(ctx context.Context, req *seqapi.DeleteAsyncSear }, ) - resp, err := a.asyncSearches.DeleteAsyncSearch(ctx, req) + resp, err := asyncSearches.DeleteAsyncSearch(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/events.go b/internal/api/seqapi/v1/grpc/events.go index a4665b9..a604a95 100644 --- a/internal/api/seqapi/v1/grpc/events.go +++ b/internal/api/seqapi/v1/grpc/events.go @@ -9,6 +9,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" @@ -18,7 +19,7 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_event") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) attributes := []attribute.KeyValue{ { @@ -38,16 +39,20 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap return nil, status.Error(codes.InvalidArgument, err.Error()) } - if cached, err := a.inmemWithRedisCache.Get(ctx, req.Id); err == nil { - event := &seqapi.Event{} - if err = proto.Unmarshal([]byte(cached), event); err == nil { - if params.masker != nil { - params.masker.Mask(event.Data) + cacheKey := params.eventsCacheKeyPrefix + req.Id + if params.eventsCache != nil { + if cached, err := params.eventsCache.Get(ctx, cacheKey); err == nil { + event := &seqapi.Event{} + if err = proto.Unmarshal([]byte(cached), event); err == nil { + if params.masker != nil { + params.masker.Mask(event.Data) + } + return &seqapi.GetEventResponse{Event: event}, nil } - return &seqapi.GetEventResponse{Event: event}, nil + logger.Error("failed to unmarshal cached event proto", zap.String("id", req.Id), zap.Error(err)) } - logger.Error("failed to unmarshal cached event proto", zap.String("id", req.Id), zap.Error(err)) } + resp, err := params.client.GetEvent(ctx, req) if err != nil { return nil, err @@ -57,10 +62,12 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap params.masker.Mask(resp.Event.Data) } - if data, err := proto.Marshal(resp.Event); err == nil { - _ = a.inmemWithRedisCache.SetWithTTL(ctx, req.Id, string(data), params.options.EventsCacheTTL) - } else { - logger.Error("failed to marshal event proto for caching", zap.String("id", req.Id), zap.Error(err)) + if params.eventsCache != nil { + if data, err := proto.Marshal(resp.Event); err == nil { + _ = params.eventsCache.SetWithTTL(ctx, cacheKey, string(data), params.options.Caches.Events.TTL) + } else { + logger.Error("failed to marshal event proto for caching", zap.String("id", req.Id), zap.Error(err)) + } } return resp, nil diff --git a/internal/api/seqapi/v1/grpc/events_test.go b/internal/api/seqapi/v1/grpc/events_test.go index 3f10c1f..cf84867 100644 --- a/internal/api/seqapi/v1/grpc/events_test.go +++ b/internal/api/seqapi/v1/grpc/events_test.go @@ -14,7 +14,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -53,7 +53,7 @@ func TestGetEvent(t *testing.T) { Event: event1, }, cacheArgs: test.CacheMockArgs{ - Key: id1, + Key: "seqapi_event_" + id1, Value: string(event1json), Err: errSomethingWrong, }, @@ -67,7 +67,7 @@ func TestGetEvent(t *testing.T) { Event: event2, }, cacheArgs: test.CacheMockArgs{ - Key: id2, + Key: "seqapi_event_" + id2, Value: string(event2json), }, }, @@ -80,7 +80,7 @@ func TestGetEvent(t *testing.T) { Event: event3, }, cacheArgs: test.CacheMockArgs{ - Key: id3, + Key: "seqapi_event_" + id3, Value: string(event3json), Err: errSomethingWrong, }, @@ -91,7 +91,7 @@ func TestGetEvent(t *testing.T) { Id: id4, }, cacheArgs: test.CacheMockArgs{ - Key: id4, + Key: "seqapi_event_" + id4, Err: errSomethingWrong, }, clientErr: errors.New("client error"), @@ -104,8 +104,10 @@ func TestGetEvent(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Events: &config.HandlerCache{TTL: cacheTTL}, + }, }, }, } @@ -369,9 +371,11 @@ func TestGetEventWithMasking(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, - Masking: tt.maskingCfg, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Events: &config.HandlerCache{TTL: cacheTTL}, + }, + Masking: tt.maskingCfg, }, }, } @@ -379,7 +383,7 @@ func TestGetEventWithMasking(t *testing.T) { ctrl := gomock.NewController(t) cacheMock := mock_cache.NewMockCache(ctrl) cacheArgs := test.CacheMockArgs{ - Key: curEID, + Key: "seqapi_event_" + curEID, Value: string(curEData.eventJson), } if !tt.isCached { diff --git a/internal/api/seqapi/v1/grpc/fetch_async_search_result.go b/internal/api/seqapi/v1/grpc/fetch_async_search_result.go index 3ba0fd4..f96eaa8 100644 --- a/internal/api/seqapi/v1/grpc/fetch_async_search_result.go +++ b/internal/api/seqapi/v1/grpc/fetch_async_search_result.go @@ -15,7 +15,8 @@ import ( ) func (a *API) FetchAsyncSearchResult(ctx context.Context, req *seqapi.FetchAsyncSearchResultRequest) (*seqapi.FetchAsyncSearchResultResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -46,7 +47,7 @@ func (a *API) FetchAsyncSearchResult(ctx context.Context, req *seqapi.FetchAsync return nil, status.Error(codes.InvalidArgument, "invalid search_id") } - resp, err := a.asyncSearches.FetchAsyncSearchResult(ctx, req) + resp, err := asyncSearches.FetchAsyncSearchResult(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/fields.go b/internal/api/seqapi/v1/grpc/fields.go index bae2836..281756b 100644 --- a/internal/api/seqapi/v1/grpc/fields.go +++ b/internal/api/seqapi/v1/grpc/fields.go @@ -2,21 +2,60 @@ package grpc import ( "context" + "time" "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) +type fieldsCache struct { + cache cache.Cache +} + +func newFieldsCache(c cache.Cache) *fieldsCache { + return &fieldsCache{cache: c} +} + +func (c *fieldsCache) get(ctx context.Context, key string) ([]*seqapi.Field, bool) { + data, err := c.cache.Get(ctx, key) + if err != nil { + return nil, false + } + + var resp seqapi.GetFieldsResponse + if err := proto.Unmarshal([]byte(data), &resp); err != nil { + logger.Error("can't unmarshal cached fields", zap.Error(err)) + return nil, false + } + + return resp.GetFields(), true +} + +func (c *fieldsCache) set(ctx context.Context, key string, fields []*seqapi.Field, ttl time.Duration) { + data, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: fields}) + if err != nil { + logger.Error("can't marshal fields for cache", zap.Error(err)) + return + } + + if err := c.cache.SetWithTTL(ctx, key, string(data), ttl); err != nil { + logger.Error("can't set fields to cache", zap.Error(err)) + } +} + func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seqapi.GetFieldsResponse, error) { ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_fields") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) @@ -33,10 +72,9 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq return resp, nil } - fields, cached, isActual := params.fieldsCache.getFields() - if cached && isActual { + if cachedFields, ok := params.fieldsCache.get(ctx, params.fieldsCacheKey); ok { return &seqapi.GetFieldsResponse{ - Fields: fields, + Fields: cachedFields, SystemFields: params.systemFields, PinnedFields: params.pinnedFields, }, nil @@ -44,34 +82,21 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq resp, err := params.client.GetFields(ctx, req) if err != nil { - if cached { - logger.Error("can't get fields; use cached fields", zap.Error(err)) - return &seqapi.GetFieldsResponse{ - Fields: fields, - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, - }, nil - } - return nil, err } - params.fieldsCache.setFields(resp.GetFields()) + params.fieldsCache.set(ctx, params.fieldsCacheKey, resp.GetFields(), params.options.Caches.Fields.TTL) resp.SystemFields = params.systemFields resp.PinnedFields = params.pinnedFields return resp, nil } func (a *API) GetPinnedFields(ctx context.Context, _ *seqapi.GetFieldsRequest) (*seqapi.GetFieldsResponse, error) { - ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_fields") - defer span.End() - - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - return &seqapi.GetFieldsResponse{ Fields: params.pinnedFields, }, nil diff --git a/internal/api/seqapi/v1/grpc/fields_test.go b/internal/api/seqapi/v1/grpc/fields_test.go index f52149d..0cdc9a6 100644 --- a/internal/api/seqapi/v1/grpc/fields_test.go +++ b/internal/api/seqapi/v1/grpc/fields_test.go @@ -11,7 +11,9 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/pkg/cache" + mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -121,7 +123,7 @@ func TestGetFields(t *testing.T) { SeqDB: seqDbMock, }, Cfg: config.SeqAPI{ - SeqAPIOptions: &tt.cfg, + Options: tt.cfg, }, } @@ -136,66 +138,70 @@ func TestGetFields(t *testing.T) { func TestGetFieldsCached(t *testing.T) { var ( - ttl = 10 * time.Millisecond + ttl = 1 * time.Hour + cacheKey = "seqapi_fields" ) - responses := []*seqapi.GetFieldsResponse{ - { - Fields: []*seqapi.Field{ - { - Name: "n1", - Type: seqapi.FieldType_keyword, - }, - { - Name: "n2", - Type: seqapi.FieldType_text, - }, + + responses := &seqapi.GetFieldsResponse{ + Fields: []*seqapi.Field{ + { + Name: "n1", + Type: seqapi.FieldType_keyword, }, - }, - { - Fields: []*seqapi.Field{ - { - Name: "qwe", - Type: seqapi.FieldType_text, - }, + { + Name: "n2", + Type: seqapi.FieldType_text, }, }, } + cached, err := proto.Marshal(responses) + require.NoError(t, err) + ctrl := gomock.NewController(t) seqDbMock := mock_seqdb.NewMockClient(ctrl) - - for _, r := range responses { - seqDbMock.EXPECT(). - GetFields(gomock.Any(), nil). - Return(proto.Clone(r), nil). - Times(1) - } + cacheMock := mock_cache.NewMockCache(ctrl) + + // first call - cache miss, requested from seqdb and set to cache + cacheMock.EXPECT(). + Get(gomock.Any(), cacheKey). + Return("", cache.ErrNotFound). + Times(1) + seqDbMock.EXPECT(). + GetFields(gomock.Any(), nil). + Return(proto.Clone(responses), nil). + Times(1) + cacheMock.EXPECT(). + SetWithTTL(gomock.Any(), cacheKey, string(cached), ttl). + Return(nil). + Times(1) + + // second call - fields from cache + cacheMock.EXPECT(). + Get(gomock.Any(), cacheKey). + Return(string(cached), nil). + Times(1) seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - FieldsCacheTTL: ttl, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Fields: &config.HandlerCache{TTL: ttl}, + }, }, }, Mocks: test.Mocks{ SeqDB: seqDbMock, + Cache: cacheMock, }, } api := setupTestAPI(seqData) - for _, r := range responses { + for range 2 { resp, err := api.GetFields(context.Background(), nil) require.NoError(t, err) - require.True(t, proto.Equal(r, resp)) - - time.Sleep(ttl / 2) - - resp, err = api.GetFields(context.Background(), nil) - require.NoError(t, err) - require.True(t, proto.Equal(r, resp)) - - time.Sleep(ttl) + require.True(t, proto.Equal(responses, resp)) } } @@ -222,7 +228,7 @@ func TestGetPinnedFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ PinnedFields: tt.fields, }, }, diff --git a/internal/api/seqapi/v1/grpc/get_async_searches_list.go b/internal/api/seqapi/v1/grpc/get_async_searches_list.go index b639207..47a1fc2 100644 --- a/internal/api/seqapi/v1/grpc/get_async_searches_list.go +++ b/internal/api/seqapi/v1/grpc/get_async_searches_list.go @@ -14,7 +14,8 @@ import ( ) func (a *API) GetAsyncSearchesList(ctx context.Context, req *seqapi.GetAsyncSearchesListRequest) (*seqapi.GetAsyncSearchesListResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -50,7 +51,7 @@ func (a *API) GetAsyncSearchesList(ctx context.Context, req *seqapi.GetAsyncSear return nil, status.Error(codes.InvalidArgument, err.Error()) } - resp, err := a.asyncSearches.GetAsyncSearchesList(ctx, req) + resp, err := asyncSearches.GetAsyncSearchesList(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/get_envs_test.go b/internal/api/seqapi/v1/grpc/get_envs_test.go index 5f5a1a4..9d98f88 100644 --- a/internal/api/seqapi/v1/grpc/get_envs_test.go +++ b/internal/api/seqapi/v1/grpc/get_envs_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -20,12 +20,14 @@ func TestGetEnvs(t *testing.T) { { name: "single_env", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: &seqapi.GetEnvsResponse{ @@ -47,47 +49,57 @@ func TestGetEnvs(t *testing.T) { Envs: map[string]config.SeqAPIEnv{ "test": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, "prod": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 150, - MaxExportLimit: 250, - MaxParallelExportRequests: 3, - MaxAggregationsPerRequest: 6, - SeqCLIMaxSearchLimit: 15000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 150, + MaxExportLimit: 250, + MaxParallelExportRequests: 3, + MaxAggregationsPerRequest: 6, + SeqCLIMaxSearchLimit: 15000, + }, }, }, "cluster-10": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 150, - MaxExportLimit: 250, - MaxParallelExportRequests: 3, - MaxAggregationsPerRequest: 6, - SeqCLIMaxSearchLimit: 15000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 150, + MaxExportLimit: 250, + MaxParallelExportRequests: 3, + MaxAggregationsPerRequest: 6, + SeqCLIMaxSearchLimit: 15000, + }, }, }, "cluster-102": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, "cluster-220": { Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, }, @@ -144,8 +156,8 @@ func TestGetEnvs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() api := API{ - config: tt.cfg, - envsResponse: parseEnvs(tt.cfg), + config: &tt.cfg, + envsResponse: parseEnvs(&tt.cfg), } resp, err := api.GetEnvs(context.TODO(), &seqapi.GetEnvsRequest{}) require.NoError(t, err) diff --git a/internal/api/seqapi/v1/grpc/histogram.go b/internal/api/seqapi/v1/grpc/histogram.go index 0d82303..63d6805 100644 --- a/internal/api/seqapi/v1/grpc/histogram.go +++ b/internal/api/seqapi/v1/grpc/histogram.go @@ -7,6 +7,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -15,7 +16,7 @@ func (a *API) GetHistogram(ctx context.Context, req *seqapi.GetHistogramRequest) ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_histogram") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) attributes := []attribute.KeyValue{ { diff --git a/internal/api/seqapi/v1/grpc/limits.go b/internal/api/seqapi/v1/grpc/limits.go index 2021bcb..fb2a66d 100644 --- a/internal/api/seqapi/v1/grpc/limits.go +++ b/internal/api/seqapi/v1/grpc/limits.go @@ -6,21 +6,22 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) func (a *API) GetLimits(ctx context.Context, _ *seqapi.GetLimitsRequest) (*seqapi.GetLimitsResponse, error) { - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } return &seqapi.GetLimitsResponse{ - MaxSearchLimit: params.options.MaxSearchLimit, - MaxExportLimit: params.options.MaxExportLimit, - MaxParallelExportRequests: int32(params.options.MaxParallelExportRequests), - MaxAggregationsPerRequest: int32(params.options.MaxAggregationsPerRequest), - SeqCliMaxSearchLimit: int32(params.options.SeqCLIMaxSearchLimit), + MaxSearchLimit: params.options.Limits.MaxSearchLimit, + MaxExportLimit: params.options.Limits.MaxExportLimit, + MaxParallelExportRequests: int32(params.options.Limits.MaxParallelExportRequests), + MaxAggregationsPerRequest: int32(params.options.Limits.MaxAggregationsPerRequest), + SeqCliMaxSearchLimit: int32(params.options.Limits.SeqCLIMaxSearchLimit), }, nil } diff --git a/internal/api/seqapi/v1/grpc/limits_test.go b/internal/api/seqapi/v1/grpc/limits_test.go index 30892c6..dc4f498 100644 --- a/internal/api/seqapi/v1/grpc/limits_test.go +++ b/internal/api/seqapi/v1/grpc/limits_test.go @@ -8,7 +8,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -21,12 +21,14 @@ func TestGetLimits(t *testing.T) { { name: "ok", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: &seqapi.GetLimitsResponse{ diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan.go b/internal/api/seqapi/v1/grpc/logs_lifespan.go index d54eb89..9648325 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan.go @@ -14,6 +14,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/grpcutil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/lifespan" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -24,26 +25,25 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_logs_lifespan") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetParams(env) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - cacheKey := params.options.LogsLifespanCacheKey + if params.logsLifespanCache != nil { + if countStr, err := params.logsLifespanCache.Get(ctx, params.logsLifespanCacheKey); err == nil { + count, err := strconv.Atoi(countStr) + if err == nil { + res := time.Duration(count) * lifespan.MeasureUnit + logger.Debug("got logs lifespan from cache") + return &seqapi.GetLogsLifespanResponse{Lifespan: durationpb.New(res)}, nil + } - if countStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { - count := 0 - count, err = strconv.Atoi(countStr) - if err == nil { - res := time.Duration(count) * lifespan.MeasureUnit - logger.Debug("got logs lifespan from cache") - return &seqapi.GetLogsLifespanResponse{Lifespan: durationpb.New(res)}, nil + logger.Error("can't parse logs lifespan", zap.Error(err)) + } else if !errors.Is(err, cache.ErrNotFound) { + logger.Error("can't get logs lifespan from cache", zap.Error(err)) } - - logger.Error("can't parse logs lifespan", zap.Error(err)) - } else if !errors.Is(err, cache.ErrNotFound) { - logger.Error("can't get logs lifespan from cache", zap.Error(err)) } clientStatus, err := params.client.Status(ctx, &seqapi.StatusRequest{}) @@ -58,9 +58,11 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ count := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) res := time.Duration(count) * lifespan.MeasureUnit - err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(count), params.options.LogsLifespanCacheTTL) - if err != nil { - logger.Error("can't set logs lifespan to cache", zap.Error(err)) + if params.logsLifespanCache != nil { + err = params.logsLifespanCache.SetWithTTL(ctx, params.logsLifespanCacheKey, strconv.Itoa(count), params.options.Caches.LogsLifespan.TTL) + if err != nil { + logger.Error("can't set logs lifespan to cache", zap.Error(err)) + } } logger.Debug("got logs lifespan from seq-proxy") diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index 9b2fc2f..e71d99f 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -13,7 +13,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" @@ -23,7 +23,7 @@ import ( func TestGetLogsLifespan(t *testing.T) { var ( resultStr = "36000" // 10(h) * 60(min/h) * 60(sec/min) - cacheKey = "logs_lifespan" + cacheKey = "seqapi_logs_lifespan" result = 10 * time.Hour cacheTTL = time.Minute ) @@ -106,9 +106,10 @@ func TestGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - LogsLifespanCacheKey: cacheKey, - LogsLifespanCacheTTL: cacheTTL, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + LogsLifespan: &config.HandlerCache{TTL: cacheTTL}, + }, }, }, } diff --git a/internal/api/seqapi/v1/grpc/search.go b/internal/api/seqapi/v1/grpc/search.go index 7fb45fb..943c979 100644 --- a/internal/api/seqapi/v1/grpc/search.go +++ b/internal/api/seqapi/v1/grpc/search.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/seqapi/v1/aggregation_ts" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -18,7 +19,7 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se ctx, span := tracing.StartSpan(ctx, "seqapi_v1_search") defer span.End() - env := a.GetEnvFromContext(ctx) + env := types.GetEnvKey(ctx) spanAttributes := []attribute.KeyValue{ { @@ -86,16 +87,16 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se span.SetAttributes(spanAttributes...) - if err := api_error.CheckSearchLimit(req.Limit, params.options.MaxSearchLimit); err != nil { + if err := api_error.CheckSearchLimit(req.Limit, params.options.Limits.MaxSearchLimit); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - if err := api_error.CheckAggregationsCount(len(req.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(req.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } if err := api_error.CheckForOffsetConflict(req.Offset, req.OffsetId); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - if err := api_error.CheckSearchOffsetLimit(req.Offset, params.options.MaxSearchOffsetLimit); err != nil { + if err := api_error.CheckSearchOffsetLimit(req.Offset, params.options.Limits.MaxSearchOffset); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -105,7 +106,7 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se continue } if err := api_error.CheckAggregationTsInterval(*agg.Interval, fromRaw, toRaw, - params.options.MaxBucketsPerAggregationTs, + params.options.Limits.MaxBucketsPerAggregationTs, ); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -116,7 +117,7 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se return nil, err } - if resp.Total > params.options.MaxSearchTotalLimit { + if resp.Total > params.options.Limits.MaxSearchTotal { resp.Error = &seqapi.Error{ Code: seqapi.ErrorCode_ERROR_CODE_QUERY_TOO_HEAVY, Message: api_error.ErrQueryTooHeavy.Error(), diff --git a/internal/api/seqapi/v1/grpc/search_test.go b/internal/api/seqapi/v1/grpc/search_test.go index b818531..273f06e 100644 --- a/internal/api/seqapi/v1/grpc/search_test.go +++ b/internal/api/seqapi/v1/grpc/search_test.go @@ -12,7 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -62,9 +62,11 @@ func TestSearch(t *testing.T) { }, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 5, + }, }, }), }, @@ -81,8 +83,10 @@ func TestSearch(t *testing.T) { Limit: 10, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }, apiErr: true, @@ -98,9 +102,11 @@ func TestSearch(t *testing.T) { }, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 2, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 2, + }, }, }), apiErr: true, @@ -115,9 +121,11 @@ func TestSearch(t *testing.T) { Offset: 11, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxSearchOffset: 10, + }, }, }), apiErr: true, @@ -145,9 +153,11 @@ func TestSearch(t *testing.T) { }, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchTotalLimit: int64(limit), + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxSearchTotal: int64(limit), + }, }, }), }, @@ -161,8 +171,10 @@ func TestSearch(t *testing.T) { Offset: 0, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), clientErr: errSomethingWrong, diff --git a/internal/api/seqapi/v1/grpc/start_async_search.go b/internal/api/seqapi/v1/grpc/start_async_search.go index 86641ed..d55c316 100644 --- a/internal/api/seqapi/v1/grpc/start_async_search.go +++ b/internal/api/seqapi/v1/grpc/start_async_search.go @@ -15,7 +15,8 @@ import ( ) func (a *API) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSearchRequest) (*seqapi.StartAsyncSearchResponse, error) { - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[types.GetEnvKey(ctx)] + if !ok { return nil, status.Error(codes.Unimplemented, types.ErrAsyncSearchesDisabled.Error()) } @@ -64,7 +65,7 @@ func (a *API) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSearch span.SetAttributes(spanAttributes...) - resp, err := a.asyncSearches.StartAsyncSearch(ctx, req) + resp, err := asyncSearches.StartAsyncSearch(ctx, req) if err != nil { return nil, grpcutil.ProcessError(err) } diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index a088897..6987adc 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -5,7 +5,7 @@ import ( "time" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) @@ -18,21 +18,32 @@ var ( ) func setupTestAPI(data test.APITestData) *API { - // when test cases don't explicitly provide configuration - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} + seqDBClients := make(map[string]seqdb.Client, max(1, len(data.Cfg.Envs))) + asyncSearhes := make(map[string]asyncsearches.Service, max(1, len(data.Cfg.Envs))) + + if len(data.Cfg.Envs) == 0 { + seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB + asyncSearhes[""] = data.Mocks.AsyncSearchesSvc + } + for envName, envConfig := range data.Cfg.Envs { + seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB + asyncSearhes[envName] = data.Mocks.AsyncSearchesSvc } - seqDBClients := make(map[string]seqdb.Client) - seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB - for _, envConfig := range data.Cfg.Envs { - seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB + if data.Mocks.AsyncSearchesSvc == nil { + asyncSearhes = nil } - var asyncSvc asyncsearches.Service - if data.Mocks.AsyncSearchesSvc != nil { - asyncSvc = data.Mocks.AsyncSearchesSvc + api := New(&data.Cfg, seqDBClients, asyncSearhes) + + if data.Mocks.Cache != nil { + for env, params := range api.paramsByEnv { + params.eventsCache = data.Mocks.Cache + params.logsLifespanCache = data.Mocks.Cache + params.fieldsCache = newFieldsCache(data.Mocks.Cache) + api.paramsByEnv[env] = params + } } - return New(data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) + return api } diff --git a/internal/api/seqapi/v1/http/aggregation.go b/internal/api/seqapi/v1/http/aggregation.go index 1e8273d..3a5dab8 100644 --- a/internal/api/seqapi/v1/http/aggregation.go +++ b/internal/api/seqapi/v1/http/aggregation.go @@ -11,6 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -40,7 +41,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { } aggsRaw, _ := json.Marshal(httpReq.Aggregations) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) attributes := []attribute.KeyValue{ { @@ -81,7 +82,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { return } - if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { wr.Error(err, http.StatusBadRequest) return } diff --git a/internal/api/seqapi/v1/http/aggregation_test.go b/internal/api/seqapi/v1/http/aggregation_test.go index 6a675f4..150063e 100644 --- a/internal/api/seqapi/v1/http/aggregation_test.go +++ b/internal/api/seqapi/v1/http/aggregation_test.go @@ -11,7 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -63,8 +63,10 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: false, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -106,8 +108,10 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: false, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -168,8 +172,10 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: false, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -203,8 +209,10 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: true, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -218,8 +226,10 @@ func TestServeGetAggregation(t *testing.T) { }, wantErr: true, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 2, + }, }, }, }, @@ -242,8 +252,10 @@ func TestServeGetAggregation(t *testing.T) { }, wantErr: true, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, diff --git a/internal/api/seqapi/v1/http/aggregation_ts.go b/internal/api/seqapi/v1/http/aggregation_ts.go index f4d5ac5..130d9e7 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts.go +++ b/internal/api/seqapi/v1/http/aggregation_ts.go @@ -13,6 +13,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/aggregation_ts" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -34,7 +35,7 @@ func (a *API) serveGetAggregationTs(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) var httpReq getAggregationTsRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -75,13 +76,13 @@ func (a *API) serveGetAggregationTs(w http.ResponseWriter, r *http.Request) { return } - if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { wr.Error(err, http.StatusBadRequest) return } for _, agg := range httpReq.Aggregations { if err := api_error.CheckAggregationTsInterval(agg.Interval, httpReq.From, httpReq.To, - params.options.MaxBucketsPerAggregationTs, + params.options.Limits.MaxBucketsPerAggregationTs, ); err != nil { wr.Error(err, http.StatusBadRequest) return diff --git a/internal/api/seqapi/v1/http/aggregation_ts_test.go b/internal/api/seqapi/v1/http/aggregation_ts_test.go index 9c4f86c..852b3e8 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts_test.go +++ b/internal/api/seqapi/v1/http/aggregation_ts_test.go @@ -17,7 +17,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -106,9 +106,11 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"test_count1":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count1":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count1":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}},{"data":{"result":[{"metric":{"test_count2":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count2":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count2":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -162,9 +164,11 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"test_count1":"test1"},"values":[{"timestamp":1695637231,"value":3}]},{"metric":{"test_count1":"test2"},"values":[{"timestamp":1695637232,"value":6}]},{"metric":{"test_count1":"test3"},"values":[{"timestamp":1695637233,"value":9}]}]}},{"data":{"result":[{"metric":{"test_count2":"test1"},"values":[{"timestamp":1695637231,"value":2}]},{"metric":{"test_count2":"test2"},"values":[{"timestamp":1695637232,"value":4}]},{"metric":{"test_count2":"test3"},"values":[{"timestamp":1695637233,"value":6}]}],"target_bucket_rate":"2s"}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -221,9 +225,11 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"quantile":"p95","service":"test1"},"values":[{"timestamp":1695637231,"value":100}]},{"metric":{"quantile":"p99","service":"test1"},"values":[{"timestamp":1695637231,"value":150}]},{"metric":{"quantile":"p95","service":"test2"},"values":[{"timestamp":1695637232,"value":100}]},{"metric":{"quantile":"p99","service":"test2"},"values":[{"timestamp":1695637232,"value":150}]},{"metric":{"quantile":"p95","service":"test3"},"values":[{"timestamp":1695637233,"value":100}]},{"metric":{"quantile":"p99","service":"test3"},"values":[{"timestamp":1695637233,"value":150}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -247,9 +253,11 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":null,"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -263,9 +271,11 @@ func TestServeGetAggregationTs(t *testing.T) { reqBody: formatReqBody(aggregationTsQueries{{}, {}, {}}), wantStatus: http.StatusBadRequest, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, - MaxBucketsPerAggregationTs: 100, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 2, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -278,9 +288,11 @@ func TestServeGetAggregationTs(t *testing.T) { }), wantStatus: http.StatusBadRequest, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 1, + }, }, }, }, @@ -297,8 +309,10 @@ func TestServeGetAggregationTs(t *testing.T) { }, wantStatus: http.StatusInternalServerError, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxAggregationsPerRequest: 3, + }, }, }, }, diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index 4e51bbe..0fc12f3 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -10,7 +10,7 @@ import ( "github.com/gofrs/uuid" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/tokenlimiter" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" @@ -22,125 +22,102 @@ import ( ) type apiParams struct { - client seqdb.Client - options *config.SeqAPIOptions - fieldsCache *fieldsCache - masker *mask.Masker - pinnedFields fields - systemFields fields - exportLimiter *tokenlimiter.Limiter + client seqdb.Client + options *config.SeqAPIOptions + masker *mask.Masker + pinnedFields fields + systemFields fields + exportLimiter *tokenlimiter.Limiter + eventsCache cache.Cache + logsLifespanCache cache.Cache + fieldsCache *fieldsCache + eventsCacheKeyPrefix string + fieldsCacheKey string + logsLifespanCacheKey string } type API struct { - config config.SeqAPI - params apiParams - paramsByEnv map[string]apiParams - inmemWithRedisCache cache.Cache - redisCache cache.Cache - nowFn func() time.Time - asyncSearches asyncsearches.Service - envsResponse getEnvsResponse + config *config.SeqAPI + paramsByEnv map[string]apiParams + nowFn func() time.Time + asyncSearches map[string]asyncsearches.Service + envsResponse getEnvsResponse } -func New( - cfg config.SeqAPI, - seqDBСlients map[string]seqdb.Client, - inmemWithRedisCache cache.Cache, - redisCache cache.Cache, - asyncSearches asyncsearches.Service, -) *API { - var globalfCache *fieldsCache - if cfg.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.FieldsCacheTTL) - } - - globalMasker, err := mask.New(cfg.Masking) - if err != nil { - logger.Fatal("failed to init masking", zap.Error(err)) - } - - globalPinnedFields := parseFields(cfg.PinnedFields) - globalSystemFields := parseFields(cfg.SystemFields) - globalExportLimiter := tokenlimiter.New(cfg.MaxParallelExportRequests) - - var params apiParams - var paramsByEnv map[string]apiParams - +func New(cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, asyncSearches map[string]asyncsearches.Service) *API { + paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { - paramsByEnv = make(map[string]apiParams) for envName, envConfig := range cfg.Envs { - client := seqDBСlients[envConfig.SeqDB] - options := envConfig.Options - - var envfCache *fieldsCache - if options.FieldsCacheTTL > 0 { - envfCache = newFieldsCache(options.FieldsCacheTTL) - } - - var envMasker, err = mask.New(options.Masking) - if err != nil { - logger.Fatal("failed to init env masking", zap.Error(err)) - } - - envPinnedFields := parseFields(options.PinnedFields) - envSystemFields := parseFields(options.SystemFields) - envExportLimiter := tokenlimiter.New(options.MaxParallelExportRequests) - - paramsByEnv[envName] = apiParams{ - client: client, - options: options, - fieldsCache: envfCache, - masker: envMasker, - pinnedFields: envPinnedFields, - systemFields: envSystemFields, - exportLimiter: envExportLimiter, - } + paramsByEnv[envName] = newAPIParams(envName, seqDBСlients[envConfig.SeqDBID], envConfig.Options) } } else { - client, exists := seqDBСlients[config.DefaultSeqDBClientID] - if !exists { - logger.Fatal("default client not found", - zap.String("clientID", config.DefaultSeqDBClientID)) + var client seqdb.Client + for _, c := range seqDBСlients { + client = c } + paramsByEnv[""] = newAPIParams("", client, &cfg.Options) + } - params = apiParams{ - client: client, - options: cfg.SeqAPIOptions, - fieldsCache: globalfCache, - masker: globalMasker, - pinnedFields: globalPinnedFields, - systemFields: globalSystemFields, - exportLimiter: globalExportLimiter, - } + return &API{ + config: cfg, + paramsByEnv: paramsByEnv, + nowFn: time.Now, + asyncSearches: asyncSearches, + envsResponse: parseEnvs(cfg), } +} + +func newAPIParams(env string, client seqdb.Client, options *config.SeqAPIOptions) apiParams { + masker, err := mask.New(options.Masking) + if err != nil { + logger.Fatal("failed to init masking", zap.String("env", env), zap.Error(err)) + } + // for export - if len(cfg.Envs) > 0 { - for _, param := range paramsByEnv { - if param.masker != nil { - param.client.WithMasking(param.masker) - } + if masker != nil { + client.WithMasking(masker) + } + + var eventsCache, logsLifespanCache cache.Cache + if options.Caches.Events != nil { + eventsCache, _ = cache.ByID(options.Caches.Events.ID) + } + if options.Caches.LogsLifespan != nil { + logsLifespanCache, _ = cache.ByID(options.Caches.LogsLifespan.ID) + } + + var fc *fieldsCache + if options.Caches.Fields != nil { + if c, ok := cache.ByID(options.Caches.Fields.ID); ok { + fc = newFieldsCache(c) } - } else if params.masker != nil { - params.client.WithMasking(params.masker) } - return &API{ - config: cfg, - params: params, - paramsByEnv: paramsByEnv, - inmemWithRedisCache: inmemWithRedisCache, - redisCache: redisCache, - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + cacheKeyPrefix := "seqapi_" + if env != "" { + cacheKeyPrefix += env + "_" + } + + return apiParams{ + client: client, + options: options, + masker: masker, + pinnedFields: parseFields(options.PinnedFields), + systemFields: parseFields(options.SystemFields), + exportLimiter: tokenlimiter.New(options.Limits.MaxParallelExportRequests), + eventsCache: eventsCache, + logsLifespanCache: logsLifespanCache, + fieldsCache: fc, + eventsCacheKeyPrefix: cacheKeyPrefix + "event_", + fieldsCacheKey: cacheKeyPrefix + "fields", + logsLifespanCacheKey: cacheKeyPrefix + "logs_lifespan", } } func (a *API) Router() chi.Router { mux := chi.NewMux() - mux.Use(a.envInterceptor) - + // seqapi mux.Post("/aggregation", a.serveGetAggregation) mux.Post("/aggregation_ts", a.serveGetAggregationTs) mux.Get("/events/{id}", a.serveGetEvent) @@ -176,7 +153,7 @@ func parseFields(fields []config.Field) []field { return res } -func parseEnvs(cfg config.SeqAPI) getEnvsResponse { +func parseEnvs(cfg *config.SeqAPI) getEnvsResponse { var envs []envInfo if len(cfg.Envs) > 0 { // sort environment names to ensure deterministic output @@ -220,7 +197,7 @@ func parseEnvs(cfg config.SeqAPI) getEnvsResponse { envs = append(envs, createEnvInfo(name, envConfig.Options)) } } else { - envs = []envInfo{createEnvInfo("", cfg.SeqAPIOptions)} + envs = []envInfo{createEnvInfo("", &cfg.Options)} } return getEnvsResponse{Envs: envs} } @@ -237,7 +214,7 @@ const ( func (a *API) GetEnvParams(env string) (apiParams, error) { if len(a.config.Envs) == 0 { - return a.params, nil + return a.paramsByEnv[""], nil } if env == "" { @@ -279,26 +256,6 @@ func apiErrorFromProto(proto *seqapi.Error) apiError { } } -type fieldsCache struct { - ttl time.Duration - - ts time.Time - rawFields []byte -} - -func newFieldsCache(ttl time.Duration) *fieldsCache { - return &fieldsCache{ttl: ttl} -} - -func (c *fieldsCache) getFields() ([]byte, bool, bool) { - return c.rawFields, !c.ts.IsZero(), time.Since(c.ts) < c.ttl -} - -func (c *fieldsCache) setFields(rawFields []byte) { - c.rawFields = rawFields - c.ts = time.Now() -} - type asyncSearchStatus string // @name seqapi.v1.AsyncSearchStatus const ( @@ -358,10 +315,10 @@ func checkLimitOffset(limit, offset int32) error { func createEnvInfo(envName string, opts *config.SeqAPIOptions) envInfo { return envInfo{ Env: envName, - MaxSearchLimit: uint32(opts.MaxSearchLimit), - MaxExportLimit: uint32(opts.MaxExportLimit), - MaxParallelExportRequests: uint32(opts.MaxParallelExportRequests), - MaxAggregationsPerRequest: uint32(opts.MaxAggregationsPerRequest), - SeqCliMaxSearchLimit: uint32(opts.SeqCLIMaxSearchLimit), + MaxSearchLimit: uint32(opts.Limits.MaxSearchLimit), + MaxExportLimit: uint32(opts.Limits.MaxExportLimit), + MaxParallelExportRequests: uint32(opts.Limits.MaxParallelExportRequests), + MaxAggregationsPerRequest: uint32(opts.Limits.MaxAggregationsPerRequest), + SeqCliMaxSearchLimit: uint32(opts.Limits.SeqCLIMaxSearchLimit), } } diff --git a/internal/api/seqapi/v1/http/cancel_async_search.go b/internal/api/seqapi/v1/http/cancel_async_search.go index b07776d..f037a85 100644 --- a/internal/api/seqapi/v1/http/cancel_async_search.go +++ b/internal/api/seqapi/v1/http/cancel_async_search.go @@ -19,35 +19,45 @@ import ( // @ID seqapi_v1_cancel_async_search // @Tags seqapi_v1 // @Param id path string true "search id" +// @Param env query string false "Environment" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveCancelAsyncSearch(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_cancel_async_search") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_cancel_async_search") - defer span.End() - searchID := chi.URLParam(r, "id") - + // я бы в service вынес проверочку и попробовал перейти на стандартную либу uuid(она вроде с этой версии появилось) + // https://go.dev/doc/go1.27#uuid такая вот фигня if err := checkUUID(searchID); err != nil { wr.Error(err, http.StatusBadRequest) return } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "search_id", Value: attribute.StringValue(searchID), }, - ) + } + + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + + span.SetAttributes(attributes...) - _, err := a.asyncSearches.CancelAsyncSearch(ctx, &seqapi.CancelAsyncSearchRequest{SearchId: searchID}) + _, err := asyncSearches.CancelAsyncSearch(ctx, &seqapi.CancelAsyncSearchRequest{SearchId: searchID}) if err != nil { status := http.StatusInternalServerError if errors.Is(err, types.ErrPermissionDenied) { diff --git a/internal/api/seqapi/v1/http/cluster_status.go b/internal/api/seqapi/v1/http/cluster_status.go index 8f8ec13..4f1ca1b 100644 --- a/internal/api/seqapi/v1/http/cluster_status.go +++ b/internal/api/seqapi/v1/http/cluster_status.go @@ -7,6 +7,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -25,7 +26,7 @@ func (a *API) serveStatus(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { wr.Error(err, http.StatusBadRequest) diff --git a/internal/api/seqapi/v1/http/delete_async_search.go b/internal/api/seqapi/v1/http/delete_async_search.go index 28e64a7..406d78a 100644 --- a/internal/api/seqapi/v1/http/delete_async_search.go +++ b/internal/api/seqapi/v1/http/delete_async_search.go @@ -19,35 +19,44 @@ import ( // @ID seqapi_v1_delete_async_search // @Tags seqapi_v1 // @Param id path string true "search id" +// @Param env query string false "Environment" // @Success 200 {object} nil "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveDeleteAsyncSearch(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_delete_async_search") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_delete_async_search") - defer span.End() - searchID := chi.URLParam(r, "id") - + // в cancel_async описал, чтобы я с этой проверкой сделал if err := checkUUID(searchID); err != nil { wr.Error(err, http.StatusBadRequest) return } - span.SetAttributes( - attribute.KeyValue{ + attributes := []attribute.KeyValue{ + { Key: "search_id", Value: attribute.StringValue(searchID), }, - ) + } + + if env != "" { + attributes = append(attributes, attribute.String("env", env)) + } + + span.SetAttributes(attributes...) - _, err := a.asyncSearches.DeleteAsyncSearch(ctx, &seqapi.DeleteAsyncSearchRequest{SearchId: searchID}) + _, err := asyncSearches.DeleteAsyncSearch(ctx, &seqapi.DeleteAsyncSearchRequest{SearchId: searchID}) if err != nil { status := http.StatusInternalServerError if errors.Is(err, types.ErrPermissionDenied) { diff --git a/internal/api/seqapi/v1/http/events.go b/internal/api/seqapi/v1/http/events.go index 5305a57..c9c725b 100644 --- a/internal/api/seqapi/v1/http/events.go +++ b/internal/api/seqapi/v1/http/events.go @@ -10,6 +10,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" @@ -30,11 +31,9 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - + env := types.GetEnvKey(ctx) id := chi.URLParam(r, "id") - env := getEnvFromContext(ctx) - attributes := []attribute.KeyValue{ { Key: "id", @@ -54,21 +53,22 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { return } - if cached, err := a.inmemWithRedisCache.Get(ctx, id); err == nil { - e := &seqapi.Event{} - if err = proto.Unmarshal([]byte(cached), e); err == nil { - if params.masker != nil { - params.masker.Mask(e.Data) + cacheKey := params.eventsCacheKeyPrefix + id // как лучше с ним поступить не придумал + if params.eventsCache != nil { + if cached, err := params.eventsCache.Get(ctx, cacheKey); err == nil { + e := &seqapi.Event{} + if err = proto.Unmarshal([]byte(cached), e); err == nil { + if params.masker != nil { + params.masker.Mask(e.Data) + } + wr.WriteJson(getEventResponseFromProto(&seqapi.GetEventResponse{Event: e})) + return } - wr.WriteJson(getEventResponseFromProto(&seqapi.GetEventResponse{Event: e})) - return + logger.Error("failed to unmarshal cached event proto", zap.String("id", id), zap.Error(err)) } - logger.Error("failed to unmarshal cached event proto", zap.String("id", id), zap.Error(err)) } - resp, err := params.client.GetEvent(ctx, &seqapi.GetEventRequest{ - Id: id, - }) + resp, err := params.client.GetEvent(ctx, &seqapi.GetEventRequest{Id: id}) if err != nil { wr.Error(err, http.StatusInternalServerError) return @@ -78,10 +78,12 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { params.masker.Mask(resp.Event.Data) } - if data, err := proto.Marshal(resp.Event); err == nil { - _ = a.inmemWithRedisCache.SetWithTTL(ctx, id, string(data), params.options.EventsCacheTTL) - } else { - logger.Error("failed to marshal event proto for caching", zap.String("id", id), zap.Error(err)) + if params.eventsCache != nil { + if data, err := proto.Marshal(resp.Event); err == nil { + _ = params.eventsCache.SetWithTTL(ctx, cacheKey, string(data), params.options.Caches.Events.TTL) + } else { + logger.Error("failed to marshal event proto for caching", zap.String("id", id), zap.Error(err)) + } } eventResp := getEventResponseFromProto(resp) diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index c9a6d48..b1a3d3b 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -14,7 +14,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -57,7 +57,7 @@ func TestServeGetEvent(t *testing.T) { id: id1, want: getEventResponse{Event: eventFromProto(event1)}, cacheArgs: test.CacheMockArgs{ - Key: id1, + Key: "seqapi_event_" + id1, Value: string(event1json), Err: errSomethingWrong, }, @@ -75,7 +75,7 @@ func TestServeGetEvent(t *testing.T) { id: id2, want: getEventResponse{Event: eventFromProto(event2)}, cacheArgs: test.CacheMockArgs{ - Key: id2, + Key: "seqapi_event_" + id2, Value: string(event2json), }, }, @@ -84,7 +84,7 @@ func TestServeGetEvent(t *testing.T) { id: id3, want: getEventResponse{Event: eventFromProto(event3)}, cacheArgs: test.CacheMockArgs{ - Key: id3, + Key: "seqapi_event_" + id3, Value: string(event3json), Err: errSomethingWrong, }, @@ -102,7 +102,7 @@ func TestServeGetEvent(t *testing.T) { id: id4, wantErr: true, cacheArgs: test.CacheMockArgs{ - Key: id4, + Key: "seqapi_event_" + id4, Err: errSomethingWrong, }, mockArgs: &mockArgs{ @@ -113,7 +113,7 @@ func TestServeGetEvent(t *testing.T) { }, }, } - + // убрать фатал при пинге(пинговать, но не фаталиться) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() @@ -121,8 +121,10 @@ func TestServeGetEvent(t *testing.T) { ctrl := gomock.NewController(t) seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Events: &config.HandlerCache{TTL: cacheTTL}, + }, }, }, } @@ -381,16 +383,18 @@ func TestGetEventWithMasking(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, - Masking: tt.maskingCfg, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + Events: &config.HandlerCache{TTL: cacheTTL}, + }, + Masking: tt.maskingCfg, }, }, } cacheMock := mock_cache.NewMockCache(ctrl) cacheArgs := test.CacheMockArgs{ - Key: curEID, + Key: "seqapi_event_" + curEID, Value: string(curEData.eventJson), } if !tt.isCached { diff --git a/internal/api/seqapi/v1/http/export.go b/internal/api/seqapi/v1/http/export.go index 379f17b..9e10401 100644 --- a/internal/api/seqapi/v1/http/export.go +++ b/internal/api/seqapi/v1/http/export.go @@ -32,8 +32,7 @@ func (a *API) serveExport(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) userStr := "_" if userName, err := types.GetUserKey(ctx); err == nil { @@ -104,9 +103,9 @@ func (a *API) serveExport(w http.ResponseWriter, r *http.Request) { } defer params.exportLimiter.Fill(userStr) - if httpReq.Limit > params.options.MaxExportLimit { + if httpReq.Limit > params.options.Limits.MaxExportLimit { wr.Error(fmt.Errorf("too many events are requested: count=%d, max=%d", - httpReq.Limit, params.options.MaxExportLimit), + httpReq.Limit, params.options.Limits.MaxExportLimit), http.StatusBadRequest) return } diff --git a/internal/api/seqapi/v1/http/export_async_search.go b/internal/api/seqapi/v1/http/export_async_search.go index 493b1b2..f71a506 100644 --- a/internal/api/seqapi/v1/http/export_async_search.go +++ b/internal/api/seqapi/v1/http/export_async_search.go @@ -26,18 +26,18 @@ import ( // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_export_async_search_http") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_export_async_search_http") - defer span.End() - - env := getEnvFromContext(ctx) - userStr := "_" if userName, err := types.GetUserKey(ctx); err == nil { userStr = userName @@ -105,9 +105,9 @@ func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { } defer params.exportLimiter.Fill(userStr) - if httpReq.Limit > params.options.MaxExportLimit { + if httpReq.Limit > params.options.Limits.MaxExportLimit { wr.Error(fmt.Errorf("too many events are requested: count=%d, max=%d", - httpReq.Limit, params.options.MaxExportLimit), + httpReq.Limit, params.options.Limits.MaxExportLimit), http.StatusBadRequest) return } @@ -123,7 +123,7 @@ func (a *API) serveExportAsyncSearch(w http.ResponseWriter, r *http.Request) { return } - if err := a.asyncSearches.ExportAsyncSearch(ctx, httpReq.toProto(), cw); err != nil { + if err := asyncSearches.ExportAsyncSearch(ctx, httpReq.toProto(), cw); err != nil { wr.Error(err, http.StatusInternalServerError) return } diff --git a/internal/api/seqapi/v1/http/export_async_search_test.go b/internal/api/seqapi/v1/http/export_async_search_test.go index 564425a..7e6150b 100644 --- a/internal/api/seqapi/v1/http/export_async_search_test.go +++ b/internal/api/seqapi/v1/http/export_async_search_test.go @@ -8,7 +8,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -36,9 +36,11 @@ func TestServeExportAsyncSearch(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, mockArgs: &mockArgs{ @@ -60,9 +62,11 @@ func TestServeExportAsyncSearch(t *testing.T) { Fields: []string{"field1", "field2"}, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, mockArgs: &mockArgs{ @@ -83,8 +87,10 @@ func TestServeExportAsyncSearch(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxParallelExportRequests: 0, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxParallelExportRequests: 0, + }, }, }, wantErr: true, @@ -97,9 +103,11 @@ func TestServeExportAsyncSearch(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 5, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 5, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -113,9 +121,11 @@ func TestServeExportAsyncSearch(t *testing.T) { Format: efCSV, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -128,9 +138,11 @@ func TestServeExportAsyncSearch(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -143,9 +155,11 @@ func TestServeExportAsyncSearch(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, diff --git a/internal/api/seqapi/v1/http/export_test.go b/internal/api/seqapi/v1/http/export_test.go index d203aa3..a2f92e8 100644 --- a/internal/api/seqapi/v1/http/export_test.go +++ b/internal/api/seqapi/v1/http/export_test.go @@ -10,7 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -40,9 +40,11 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, mockArgs: &mockArgs{ @@ -67,9 +69,11 @@ func TestServeExport(t *testing.T) { Fields: []string{"field1", "field2"}, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, mockArgs: &mockArgs{ @@ -94,8 +98,10 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxParallelExportRequests: 0, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxParallelExportRequests: 0, + }, }, }, wantErr: true, @@ -110,9 +116,11 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 5, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 5, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -128,9 +136,11 @@ func TestServeExport(t *testing.T) { Format: efCSV, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, @@ -145,9 +155,11 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, wantErr: true, diff --git a/internal/api/seqapi/v1/http/fetch_async_search_result.go b/internal/api/seqapi/v1/http/fetch_async_search_result.go index fb09d5a..14fa485 100644 --- a/internal/api/seqapi/v1/http/fetch_async_search_result.go +++ b/internal/api/seqapi/v1/http/fetch_async_search_result.go @@ -20,21 +20,24 @@ import ( // @Router /seqapi/v1/async_search/fetch [post] // @ID seqapi_v1_fetch_async_search_result // @Tags seqapi_v1 +// @Param env query string false "Environment" // @Param body body fetchAsyncSearchResultRequest true "Request body" // @Success 200 {object} fetchAsyncSearchResultResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveFetchAsyncSearchResult(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_fetch_async_search_result") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_fetch_async_search_result") - defer span.End() - var httpReq fetchAsyncSearchResultRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) @@ -59,6 +62,11 @@ func (a *API) serveFetchAsyncSearchResult(w http.ResponseWriter, r *http.Request Value: attribute.StringValue(string(httpReq.Order)), }, } + + if env != "" { + spanAttributes = append(spanAttributes, attribute.String("env", env)) + } + span.SetAttributes(spanAttributes...) if err := checkUUID(httpReq.SearchID); err != nil { @@ -71,7 +79,7 @@ func (a *API) serveFetchAsyncSearchResult(w http.ResponseWriter, r *http.Request return } - resp, err := a.asyncSearches.FetchAsyncSearchResult(ctx, httpReq.toProto()) + resp, err := asyncSearches.FetchAsyncSearchResult(ctx, httpReq.toProto()) if err != nil { wr.Error(err, http.StatusInternalServerError) return diff --git a/internal/api/seqapi/v1/http/fields.go b/internal/api/seqapi/v1/http/fields.go index c026603..f89a9ab 100644 --- a/internal/api/seqapi/v1/http/fields.go +++ b/internal/api/seqapi/v1/http/fields.go @@ -1,18 +1,57 @@ package http import ( - "encoding/json" + "context" "net/http" + "time" "go.uber.org/zap" + "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) +type fieldsCache struct { + cache cache.Cache +} + +func newFieldsCache(c cache.Cache) *fieldsCache { + return &fieldsCache{cache: c} +} + +func (c *fieldsCache) get(ctx context.Context, key string) ([]*seqapi.Field, bool) { + data, err := c.cache.Get(ctx, key) + if err != nil { + return nil, false + } + + var resp seqapi.GetFieldsResponse + if err := proto.Unmarshal([]byte(data), &resp); err != nil { + logger.Error("can't unmarshal cached fields", zap.Error(err)) + return nil, false + } + + return resp.GetFields(), true +} + +func (c *fieldsCache) set(ctx context.Context, key string, fields []*seqapi.Field, ttl time.Duration) { + data, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: fields}) + if err != nil { + logger.Error("can't marshal fields for cache", zap.Error(err)) + return + } + + if err := c.cache.SetWithTTL(ctx, key, string(data), ttl); err != nil { + logger.Error("can't set fields to cache", zap.Error(err)) + } +} + // serveGetFields go doc. // // @Router /seqapi/v1/fields [get] @@ -27,7 +66,7 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { wr.Error(err, http.StatusBadRequest) @@ -50,43 +89,30 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - rawFields, cached, isActual := params.fieldsCache.getFields() - if cached && isActual { - _, _ = wr.Write(rawFields) + if cachedFields, ok := params.fieldsCache.get(ctx, params.fieldsCacheKey); ok { + res := getFieldsResponse{ + Fields: fieldsFromProto(cachedFields), + SystemFields: params.systemFields, + PinnedFields: params.pinnedFields, + } + wr.WriteJson(res) return } resp, err := params.client.GetFields(ctx, &seqapi.GetFieldsRequest{}) if err != nil { - if cached { - logger.Error("can't get fields; use cached fields", zap.Error(err)) - _, _ = wr.Write(rawFields) - return - } - wr.Error(err, http.StatusInternalServerError) return } + params.fieldsCache.set(ctx, params.fieldsCacheKey, resp.GetFields(), params.options.Caches.Fields.TTL) res := getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), SystemFields: params.systemFields, PinnedFields: params.pinnedFields, } - resData, err := json.Marshal(res) - if err != nil { - if cached { - logger.Error("can't marshal fields; use cached fields", zap.Error(err)) - _, _ = wr.Write(rawFields) - return - } - - wr.Error(err, http.StatusInternalServerError) - return - } - params.fieldsCache.setFields(resData) - _, _ = wr.Write(resData) + wr.WriteJson(res) } // serveGetPinnedFields go doc. @@ -101,10 +127,12 @@ func (a *API) serveGetPinnedFields(w http.ResponseWriter, r *http.Request) { ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_pinned_fields") defer span.End() - env := getEnvFromContext(ctx) + wr := httputil.NewWriter(w) + + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { - httputil.NewWriter(w).Error(err, http.StatusBadRequest) + wr.Error(err, http.StatusBadRequest) return } diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index ace82fb..a8c5de4 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -1,21 +1,24 @@ package http import ( - "encoding/json" "net/http" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" + mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) +var fieldsCacheKey = "seqapi_fields" + func TestServeGetFields(t *testing.T) { f := fields{ {Name: "test_name1", Type: "keyword"}, @@ -26,11 +29,8 @@ func TestServeGetFields(t *testing.T) { {Name: "test_name2", Type: seqapi.FieldType_text}, } - marshal := func(data any) []byte { - raw, err := json.Marshal(data) - require.NoError(t, err) - return raw - } + cachedFields, err := proto.Marshal(&seqapi.GetFieldsResponse{Fields: fAPI}) + require.NoError(t, err) type mockArgs struct { resp *seqapi.GetFieldsResponse @@ -93,9 +93,13 @@ func TestServeGetFields(t *testing.T) { want: getFieldsResponse{ Fields: f, }, - cacheData: marshal(getFieldsResponse{Fields: f}), + cacheData: cachedFields, cfg: config.SeqAPIOptions{ - FieldsCacheTTL: time.Hour, + Caches: config.SeqAPICaches{ + Fields: &config.HandlerCache{ + TTL: time.Hour, + }, + }, }, }, { @@ -114,13 +118,22 @@ func TestServeGetFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &tt.cfg, + Options: tt.cfg, }, } seqDbMock := mock_seqdb.NewMockClient(ctrl) seqData.Mocks.SeqDB = seqDbMock + if tt.cacheData != nil { + cacheMock := mock_cache.NewMockCache(ctrl) + cacheMock.EXPECT(). + Get(gomock.Any(), fieldsCacheKey). + Return(string(tt.cacheData), nil). + Times(1) + seqData.Mocks.Cache = cacheMock + } + api := setupTestAPI(seqData) if tt.mockArgs != nil { @@ -129,9 +142,6 @@ func TestServeGetFields(t *testing.T) { Return(tt.mockArgs.resp, tt.mockArgs.err). Times(1) } - if tt.cacheData != nil && tt.cfg.FieldsCacheTTL > 0 { - api.params.fieldsCache.setFields(tt.cacheData) - } httputil.DoTestHTTPEx(t, httputil.TestDataHTTPEx[struct{}, getFieldsResponse]{ Method: http.MethodGet, @@ -172,7 +182,7 @@ func TestServeGetPinnedFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ PinnedFields: tt.fields, }, }, diff --git a/internal/api/seqapi/v1/http/get_async_searches_list.go b/internal/api/seqapi/v1/http/get_async_searches_list.go index 8e64d7c..efdc025 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list.go @@ -20,21 +20,24 @@ import ( // @Router /seqapi/v1/async_search/list [post] // @ID seqapi_v1_get_async_searches_list // @Tags seqapi_v1 +// @Param env query string false "Environment" // @Param body body getAsyncSearchesListRequest true "Request body" // @Success 200 {object} getAsyncSearchesListResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_async_searches_list") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_async_searches_list") - defer span.End() - var httpReq getAsyncSearchesListRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) @@ -63,6 +66,9 @@ func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) Value: attribute.StringValue(*httpReq.Owner), }) } + if env != "" { + spanAttributes = append(spanAttributes, attribute.String("env", env)) + } span.SetAttributes(spanAttributes...) @@ -77,7 +83,7 @@ func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) return } - resp, err := a.asyncSearches.GetAsyncSearchesList(ctx, parsedReq) + resp, err := asyncSearches.GetAsyncSearchesList(ctx, parsedReq) if err != nil { wr.Error(err, http.StatusInternalServerError) return diff --git a/internal/api/seqapi/v1/http/get_envs_test.go b/internal/api/seqapi/v1/http/get_envs_test.go index 3ede5d5..2e0fd59 100644 --- a/internal/api/seqapi/v1/http/get_envs_test.go +++ b/internal/api/seqapi/v1/http/get_envs_test.go @@ -6,7 +6,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestServeGetEnvs(t *testing.T) { @@ -18,12 +18,14 @@ func TestServeGetEnvs(t *testing.T) { { name: "single_env", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: getEnvsResponse{ @@ -42,56 +44,65 @@ func TestServeGetEnvs(t *testing.T) { { name: "ok_multiple_envs", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{}, Envs: map[string]config.SeqAPIEnv{ "cluster-220": { - SeqDB: "pro-seqdb", + SeqDBID: "pro-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 1000, - MaxExportLimit: 500, - MaxParallelExportRequests: 10, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 2000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 1000, + MaxExportLimit: 500, + MaxParallelExportRequests: 10, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 2000, + }, }, }, "cluster-10": { - SeqDB: "prod-seqdb", + SeqDBID: "prod-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 1000, - MaxExportLimit: 500, - MaxParallelExportRequests: 10, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 2000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 1000, + MaxExportLimit: 500, + MaxParallelExportRequests: 10, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 2000, + }, }, }, "cluster-102": { - SeqDB: "staging-seqdb", + SeqDBID: "staging-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, + }, }, }, "prod": { - SeqDB: "stag-seqdb", + SeqDBID: "stag-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, + }, }, }, "wyanki": { - SeqDB: "sta-seqdb", + SeqDBID: "sta-seqdb", Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, + Limits: config.SeqAPILimits{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, + }, }, }, }, diff --git a/internal/api/seqapi/v1/http/histogram.go b/internal/api/seqapi/v1/http/histogram.go index ba3bbc6..72717e7 100644 --- a/internal/api/seqapi/v1/http/histogram.go +++ b/internal/api/seqapi/v1/http/histogram.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -31,7 +32,7 @@ func (a *API) serveGetHistogram(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) var httpReq getHistogramRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { diff --git a/internal/api/seqapi/v1/http/limits.go b/internal/api/seqapi/v1/http/limits.go index b9f719a..6d04d8b 100644 --- a/internal/api/seqapi/v1/http/limits.go +++ b/internal/api/seqapi/v1/http/limits.go @@ -4,6 +4,7 @@ import ( "net/http" "github.com/ozontech/seq-ui/internal/api/httputil" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/tracing" ) @@ -20,7 +21,7 @@ func (a *API) serveGetLimits(w http.ResponseWriter, r *http.Request) { defer span.End() wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { @@ -29,18 +30,18 @@ func (a *API) serveGetLimits(w http.ResponseWriter, r *http.Request) { } httputil.NewWriter(w).WriteJson(getLimitsResponse{ - MaxSearchLimit: params.options.MaxSearchLimit, - MaxExportLimit: params.options.MaxExportLimit, - MaxParallelExportRequests: int32(params.options.MaxParallelExportRequests), - MaxAggregationsPerRequest: int32(params.options.MaxAggregationsPerRequest), - SeqCliMaxSearchLimit: int32(params.options.SeqCLIMaxSearchLimit), + MaxSearchLimit: int(params.options.Limits.MaxSearchLimit), + MaxExportLimit: int(params.options.Limits.MaxExportLimit), + MaxParallelExportRequests: params.options.Limits.MaxParallelExportRequests, + MaxAggregationsPerRequest: params.options.Limits.MaxAggregationsPerRequest, + SeqCliMaxSearchLimit: params.options.Limits.SeqCLIMaxSearchLimit, }) } type getLimitsResponse struct { - MaxSearchLimit int32 `json:"maxSearchLimit" format:"int32"` - MaxExportLimit int32 `json:"maxExportLimit" format:"int32"` - MaxParallelExportRequests int32 `json:"maxParallelExportRequests" format:"int32"` - MaxAggregationsPerRequest int32 `json:"maxAggregationsPerRequest" format:"int32"` - SeqCliMaxSearchLimit int32 `json:"seqCliMaxSearchLimit" format:"int32"` + MaxSearchLimit int `json:"maxSearchLimit"` + MaxExportLimit int `json:"maxExportLimit"` + MaxParallelExportRequests int `json:"maxParallelExportRequests"` + MaxAggregationsPerRequest int `json:"maxAggregationsPerRequest"` + SeqCliMaxSearchLimit int `json:"seqCliMaxSearchLimit"` } // @name seqapi.v1.GetLimitsResponse diff --git a/internal/api/seqapi/v1/http/limits_test.go b/internal/api/seqapi/v1/http/limits_test.go index cf339e4..321b81e 100644 --- a/internal/api/seqapi/v1/http/limits_test.go +++ b/internal/api/seqapi/v1/http/limits_test.go @@ -6,7 +6,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestServeGetLimits(t *testing.T) { @@ -21,12 +21,14 @@ func TestServeGetLimits(t *testing.T) { name: "ok", env: "default", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: getLimitsResponse{ diff --git a/internal/api/seqapi/v1/http/logs_lifespan.go b/internal/api/seqapi/v1/http/logs_lifespan.go index 50f9a98..0f55aa8 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan.go +++ b/internal/api/seqapi/v1/http/logs_lifespan.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/lifespan" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -30,27 +31,26 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) params, err := a.GetEnvParams(env) if err != nil { wr.Error(err, http.StatusBadRequest) return } - cacheKey := params.options.LogsLifespanCacheKey + if params.logsLifespanCache != nil { + if resStr, err := params.logsLifespanCache.Get(ctx, params.logsLifespanCacheKey); err == nil { + res, err := strconv.Atoi(resStr) + if err == nil { + logger.Debug("got logs lifespan from cache") + wr.WriteJson(getLogsLifespanResponse{Lifespan: res}) + return + } - if resStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { - res := 0 - res, err = strconv.Atoi(resStr) - if err == nil { - logger.Debug("got logs lifespan from cache") - wr.WriteJson(getLogsLifespanResponse{Lifespan: int64(res)}) - return + logger.Error("can't parse logs lifespan", zap.Error(err)) + } else if !errors.Is(err, cache.ErrNotFound) { + logger.Error("can't get logs lifespan from cache", zap.Error(err)) } - - logger.Error("can't parse logs lifespan", zap.Error(err)) - } else if !errors.Is(err, cache.ErrNotFound) { - logger.Error("can't get logs lifespan from cache", zap.Error(err)) } clientStatus, err := params.client.Status(ctx, &seqapi.StatusRequest{}) @@ -66,15 +66,20 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { res := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) - err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(res), params.options.LogsLifespanCacheTTL) - if err != nil { - logger.Error("can't set logs lifespan to cache", zap.Error(err)) + if params.logsLifespanCache != nil { + if err = params.logsLifespanCache.SetWithTTL( + ctx, + params.logsLifespanCacheKey, + strconv.Itoa(res), + params.options.Caches.LogsLifespan.TTL); err != nil { + logger.Error("can't set logs lifespan to cache", zap.Error(err)) + } } logger.Debug("got logs lifespan from seq-proxy") - wr.WriteJson(getLogsLifespanResponse{Lifespan: int64(res)}) + wr.WriteJson(getLogsLifespanResponse{Lifespan: res}) } type getLogsLifespanResponse struct { - Lifespan int64 `json:"lifespan"` + Lifespan int `json:"lifespan"` } // @name seqapi.v1.GetLogsLifespanResponse diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index 0ed4b5c..616cce6 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -12,7 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" @@ -21,8 +21,8 @@ import ( func TestServeGetLogsLifespan(t *testing.T) { var ( + cacheKey = "seqapi_logs_lifespan" resultStr = "36000" // 10(h) * 60(min/h) * 60(sec/min) - cacheKey = "logs_lifespan" result = 10 * time.Hour cacheTTL = time.Minute ) @@ -103,9 +103,12 @@ func TestServeGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - LogsLifespanCacheKey: cacheKey, - LogsLifespanCacheTTL: cacheTTL, + Options: config.SeqAPIOptions{ + Caches: config.SeqAPICaches{ + LogsLifespan: &config.HandlerCache{ + TTL: cacheTTL, + }, + }, }, }, } diff --git a/internal/api/seqapi/v1/http/mw.go b/internal/api/seqapi/v1/http/mw.go deleted file mode 100644 index ef46456..0000000 --- a/internal/api/seqapi/v1/http/mw.go +++ /dev/null @@ -1,23 +0,0 @@ -package http - -import ( - "context" - "net/http" -) - -type envContextKey struct{} - -func getEnvFromContext(ctx context.Context) string { - if v := ctx.Value(envContextKey{}); v != nil { - return v.(string) - } - return "" -} - -func (a *API) envInterceptor(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - env := r.URL.Query().Get("env") - ctx := context.WithValue(r.Context(), envContextKey{}, env) - next.ServeHTTP(w, r.WithContext(ctx)) - }) -} diff --git a/internal/api/seqapi/v1/http/search.go b/internal/api/seqapi/v1/http/search.go index fd19854..5e330fc 100644 --- a/internal/api/seqapi/v1/http/search.go +++ b/internal/api/seqapi/v1/http/search.go @@ -12,6 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -32,7 +33,7 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { wr := httputil.NewWriter(w) - env := getEnvFromContext(ctx) + env := types.GetEnvKey(ctx) var httpReq searchRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -109,11 +110,11 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { span.SetAttributes(spanAttributes...) - if err := api_error.CheckSearchLimit(httpReq.Limit, params.options.MaxSearchLimit); err != nil { + if err := api_error.CheckSearchLimit(httpReq.Limit, params.options.Limits.MaxSearchLimit); err != nil { wr.Error(err, http.StatusBadRequest) return } - if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.MaxAggregationsPerRequest); err != nil { + if err := api_error.CheckAggregationsCount(len(httpReq.Aggregations), params.options.Limits.MaxAggregationsPerRequest); err != nil { wr.Error(err, http.StatusBadRequest) return } @@ -121,7 +122,7 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { wr.Error(err, http.StatusBadRequest) return } - if err := api_error.CheckSearchOffsetLimit(httpReq.Offset, params.options.MaxSearchOffsetLimit); err != nil { + if err := api_error.CheckSearchOffsetLimit(httpReq.Offset, params.options.Limits.MaxSearchOffset); err != nil { wr.Error(err, http.StatusBadRequest) return } @@ -132,7 +133,7 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { return } - if resp.Total > params.options.MaxSearchTotalLimit { + if resp.Total > params.options.Limits.MaxSearchTotal { resp.Error = &seqapi.Error{ Code: seqapi.ErrorCode_ERROR_CODE_QUERY_TOO_HEAVY, Message: api_error.ErrQueryTooHeavy.Error(), @@ -169,8 +170,8 @@ type searchRequest struct { Query string `json:"query"` From time.Time `json:"from" format:"date-time"` To time.Time `json:"to" format:"date-time"` - Limit int32 `json:"limit" format:"int32"` - Offset int32 `json:"offset" format:"int32"` + Limit int32 `json:"limit"` + Offset int32 `json:"offset"` WithTotal bool `json:"withTotal"` Aggregations aggregationQueries `json:"aggregations"` Histogram struct { @@ -186,8 +187,8 @@ func (r searchRequest) toProto() *seqapi.SearchRequest { Query: r.Query, From: timestamppb.New(r.From), To: timestamppb.New(r.To), - Limit: r.Limit, - Offset: r.Offset, + Limit: int32(r.Limit), + Offset: int32(r.Offset), OffsetId: r.OffsetID, WithTotal: r.WithTotal, Downsample: r.Downsample, diff --git a/internal/api/seqapi/v1/http/search_test.go b/internal/api/seqapi/v1/http/search_test.go index c22272f..fb2806f 100644 --- a/internal/api/seqapi/v1/http/search_test.go +++ b/internal/api/seqapi/v1/http/search_test.go @@ -11,7 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -51,8 +51,10 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -87,8 +89,10 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -124,8 +128,10 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -163,8 +169,10 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -211,8 +219,10 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -255,8 +265,10 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -289,8 +301,10 @@ func TestServeSearch(t *testing.T) { PartialResponse: true, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), mockArgs: &mockArgs{ @@ -330,8 +344,10 @@ func TestServeSearch(t *testing.T) { Limit: 10, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }, wantErr: true, @@ -346,9 +362,11 @@ func TestServeSearch(t *testing.T) { Aggregations: aggregationQueries{{}, {}, {}}, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 2, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 2, + }, }, }, wantErr: true, @@ -363,9 +381,11 @@ func TestServeSearch(t *testing.T) { Offset: 11, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxSearchOffset: 10, + }, }, }), wantErr: true, @@ -386,9 +406,11 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + MaxSearchOffset: 10, + }, }, }), mockArgs: &mockArgs{ @@ -419,8 +441,10 @@ func TestServeSearch(t *testing.T) { Limit: 3, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + Options: config.SeqAPIOptions{ + Limits: config.SeqAPILimits{ + MaxSearchLimit: 5, + }, }, }), wantErr: true, diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index 1096ec8..31543e0 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -22,20 +22,29 @@ import ( // @Router /seqapi/v1/async_search/start [post] // @ID seqapi_v1_start_async_search // @Tags seqapi_v1 +// @Param env query string false "Environment" // @Param body body startAsyncSearchRequest true "Request body" // @Success 200 {object} startAsyncSearchResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" // @Security bearer func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_start_async_search") + defer span.End() + wr := httputil.NewWriter(w) + env := types.GetEnvKey(ctx) - if a.asyncSearches == nil { + asyncSearches, ok := a.asyncSearches[env] + if !ok { wr.Error(types.ErrAsyncSearchesDisabled, http.StatusBadRequest) return } - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_start_async_search") - defer span.End() + params, err := a.GetEnvParams(env) + if err != nil { + httputil.NewWriter(w).Error(err, http.StatusBadRequest) + return + } var httpReq startAsyncSearchRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { @@ -54,7 +63,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { continue } if err := api_error.CheckAggregationTsInterval(agg.Interval, httpReq.From, httpReq.To, - a.config.MaxBucketsPerAggregationTs, + params.options.Limits.MaxBucketsPerAggregationTs, ); err != nil { wr.Error(err, http.StatusBadRequest) return @@ -100,10 +109,13 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { Value: attribute.StringValue(string(aggsRaw)), }) } + if env != "" { + spanAttributes = append(spanAttributes, attribute.String("env", env)) + } span.SetAttributes(spanAttributes...) - resp, err := a.asyncSearches.StartAsyncSearch(ctx, httpReq.toProto(parsedRetention)) + resp, err := asyncSearches.StartAsyncSearch(ctx, httpReq.toProto(parsedRetention)) if err != nil { wr.Error(err, http.StatusInternalServerError) return diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 0f24d93..43dadde 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -9,7 +9,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) @@ -23,23 +23,34 @@ var ( ) func setupTestAPI(data test.APITestData) *API { - // when test cases don't explicitly provide configuration. - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} + seqDBClients := make(map[string]seqdb.Client, max(1, len(data.Cfg.Envs))) + asyncSearhes := make(map[string]asyncsearches.Service, max(1, len(data.Cfg.Envs))) + + if len(data.Cfg.Envs) == 0 { + seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB + asyncSearhes[""] = data.Mocks.AsyncSearchesSvc + } + for envName, envConfig := range data.Cfg.Envs { + seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB + asyncSearhes[envName] = data.Mocks.AsyncSearchesSvc } - seqDBClients := make(map[string]seqdb.Client) - seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB - for _, envConfig := range data.Cfg.Envs { - seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB + if data.Mocks.AsyncSearchesSvc == nil { + asyncSearhes = nil } - var asyncSvc asyncsearches.Service - if data.Mocks.AsyncSearchesSvc != nil { - asyncSvc = data.Mocks.AsyncSearchesSvc + api := New(&data.Cfg, seqDBClients, asyncSearhes) + + if data.Mocks.Cache != nil { + for env, params := range api.paramsByEnv { + params.eventsCache = data.Mocks.Cache + params.logsLifespanCache = data.Mocks.Cache + params.fieldsCache = newFieldsCache(data.Mocks.Cache) + api.paramsByEnv[env] = params + } } - return New(data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) + return api } func withQueryParamID(h http.HandlerFunc, id string) http.HandlerFunc { diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index f1a08c4..82d337d 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -5,8 +5,7 @@ import ( grpc_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/grpc" http_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/http" - "github.com/ozontech/seq-ui/internal/app/config" - "github.com/ozontech/seq-ui/internal/pkg/cache" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) @@ -16,16 +15,10 @@ type SeqAPI struct { httpAPI *http_api.API } -func New( - cfg config.SeqAPI, - seqDB map[string]seqdb.Client, - inmemWithRedisCache cache.Cache, - redisCache cache.Cache, - asyncSearches asyncsearches.Service, -) *SeqAPI { +func New(cfg *config.SeqAPI, seqDBClients map[string]seqdb.Client, asyncSearches map[string]asyncsearches.Service) *SeqAPI { return &SeqAPI{ - grpcAPI: grpc_api.New(cfg, seqDB, inmemWithRedisCache, redisCache, asyncSearches), - httpAPI: http_api.New(cfg, seqDB, inmemWithRedisCache, redisCache, asyncSearches), + grpcAPI: grpc_api.New(cfg, seqDBClients, asyncSearches), + httpAPI: http_api.New(cfg, seqDBClients, asyncSearches), } } diff --git a/internal/api/seqapi/v1/test/data.go b/internal/api/seqapi/v1/test/data.go index 0ab1d44..6ca4084 100644 --- a/internal/api/seqapi/v1/test/data.go +++ b/internal/api/seqapi/v1/test/data.go @@ -6,7 +6,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" mock_asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches/mock" @@ -111,22 +111,22 @@ func MakeAggregations(aggCount, bucketCount int, opts *MakeAggOpts) []*seqapi.Ag } func SetCfgDefaults(cfg config.SeqAPI) config.SeqAPI { - if cfg.MaxAggregationsPerRequest <= 0 { - cfg.MaxAggregationsPerRequest = 1 + if cfg.Options.Limits.MaxAggregationsPerRequest <= 0 { + cfg.Options.Limits.MaxAggregationsPerRequest = 1 } - if cfg.MaxParallelExportRequests <= 0 { - cfg.MaxParallelExportRequests = 1 + if cfg.Options.Limits.MaxParallelExportRequests <= 0 { + cfg.Options.Limits.MaxParallelExportRequests = 1 } - if cfg.MaxSearchTotalLimit <= 0 { - cfg.MaxSearchTotalLimit = 1000000 + if cfg.Options.Limits.MaxSearchTotal <= 0 { + cfg.Options.Limits.MaxSearchTotal = 1000000 } - if cfg.MaxSearchOffsetLimit <= 0 { - cfg.MaxSearchOffsetLimit = 1000000 + if cfg.Options.Limits.MaxSearchOffset <= 0 { + cfg.Options.Limits.MaxSearchOffset = 1000000 } for envName, envConfig := range cfg.Envs { if envConfig.Options == nil { - envConfig.Options = cfg.SeqAPIOptions + envConfig.Options = &cfg.Options cfg.Envs[envName] = envConfig } } diff --git a/internal/api/userprofile/v1/grpc/user_profiles_test.go b/internal/api/userprofile/v1/grpc/user_profiles_test.go index 5558d95..0ffd540 100644 --- a/internal/api/userprofile/v1/grpc/user_profiles_test.go +++ b/internal/api/userprofile/v1/grpc/user_profiles_test.go @@ -81,7 +81,7 @@ func TestGetUserProfile(t *testing.T) { Times(1) if tt.mockArgs.err == nil { - profiles.InitProfiles(mockedSvc.GetOrCreateUserProfile) + profiles.Init(mockedSvc.GetOrCreateUserProfile) profiles.SetID(tt.mockArgs.resp.UserName, tt.mockArgs.resp.ID) } } diff --git a/internal/api/userprofile/v1/http/user_profiles_test.go b/internal/api/userprofile/v1/http/user_profiles_test.go index 4e0fafe..6c12005 100644 --- a/internal/api/userprofile/v1/http/user_profiles_test.go +++ b/internal/api/userprofile/v1/http/user_profiles_test.go @@ -78,7 +78,7 @@ func TestServeGetUserProfile(t *testing.T) { Times(1) if tt.mockArgs.err == nil { - profiles.InitProfiles(mockedSvc.GetOrCreateUserProfile) + profiles.Init(mockedSvc.GetOrCreateUserProfile) profiles.SetID(tt.mockArgs.resp.UserName, tt.mockArgs.resp.ID) } } diff --git a/internal/app/auth/jwt.go b/internal/app/auth/jwt.go index c6dcbc6..988d4e1 100644 --- a/internal/app/auth/jwt.go +++ b/internal/app/auth/jwt.go @@ -6,6 +6,8 @@ import ( "time" "github.com/golang-jwt/jwt/v4" + + "github.com/ozontech/seq-ui/internal/app/config/v2" ) type JWTClaims struct { @@ -23,9 +25,9 @@ type jwtProvider struct { secretKey string } -func NewJWTProvider(secretKey string) JWTProvider { +func NewJWTProvider(cfg *config.JWT) JWTProvider { return &jwtProvider{ - secretKey: secretKey, + secretKey: cfg.SecretKey, } } diff --git a/internal/app/auth/jwt_test.go b/internal/app/auth/jwt_test.go index 684739b..542a453 100644 --- a/internal/app/auth/jwt_test.go +++ b/internal/app/auth/jwt_test.go @@ -5,12 +5,14 @@ import ( "time" "github.com/stretchr/testify/require" + + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestJWTProvider(t *testing.T) { expInterval := time.Second - p1 := NewJWTProvider("secret-1") - p2 := NewJWTProvider("secret-2") + p1 := NewJWTProvider(&config.JWT{SecretKey: "secret-1"}) + p2 := NewJWTProvider(&config.JWT{SecretKey: "secret-2"}) // test valid issue token tokenName1 := "test-1" testStart := time.Now() diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 15caf35..783b19f 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -13,7 +13,7 @@ import ( "github.com/coreos/go-oidc/v3/oidc" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/tls" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" @@ -42,31 +42,22 @@ type oidcProvider struct { allowedClients []string } -func NewOIDCProvider(ctx context.Context, cfg *config.OIDC, cacheCfg config.Cache) (OIDCProvider, error) { +func NewOIDCProvider(ctx context.Context, cfg *config.OIDC) (OIDCProvider, error) { var ( err error oidcCache cache.Cache oidcCtx context.Context ) - if cfg.CacheSecretKey != "" { - logger.Info("initializing oidc cache") - oidcCache, err = cache.NewInmemoryWithRedisOrInmemory(ctx, cacheCfg) - if err != nil { - return nil, fmt.Errorf("init oidc cache: %w", err) + if cfg.CacheID != "" { + var found bool + if oidcCache, found = cache.ByID(cfg.CacheID); !found { + return nil, fmt.Errorf("oidc cache %q not found", cfg.CacheID) } } if !cfg.SkipVerify { - oidcCtx, err = newHTTPContext( - context.Background(), - httpContextCfg{ - rootCA: cfg.RootCA, - caCert: cfg.CACert, - privateKey: cfg.PrivateKey, - sslSkipVerify: cfg.SSLSkipVerify, - }, - ) + oidcCtx, err = newHTTPContext(context.Background(), cfg.TLS) if err != nil { return nil, err } @@ -178,31 +169,24 @@ func (p *oidcProvider) checkClients(clients []string) error { return fmt.Errorf("no allowed client found (clients: %v; allowed clients: %v)", clients, p.allowedClients) } -type httpContextCfg struct { - rootCA string - caCert string - privateKey string - sslSkipVerify bool -} - -func newHTTPContext(ctx context.Context, conf httpContextCfg) (context.Context, error) { +func newHTTPContext(ctx context.Context, cfg *config.TLS) (context.Context, error) { hc := &http.Client{ Timeout: oidcClientTimeout, } - if conf.isZero() { + if cfg == nil { return oidc.ClientContext(ctx, hc), nil } b := tls.NewConfigBuilder() - if conf.sslSkipVerify { - b.SetInsecureSkipVerify(conf.sslSkipVerify) - } else if !conf.sslSkipVerify && (conf.rootCA != "" || conf.caCert != "" || conf.privateKey != "") { - if conf.rootCA != "" { - if err := b.AppendCARoot(conf.rootCA); err != nil { + if cfg.Insecure { + b.SetInsecureSkipVerify(true) + } else { + if cfg.CACert != "" { + if err := b.AppendCARoot(cfg.CACert); err != nil { return nil, fmt.Errorf("can't append CA root: %w", err) } } - if conf.caCert != "" || conf.privateKey != "" { - if err := b.AppendX509KeyPair(conf.caCert, conf.privateKey); err != nil { + if cfg.ClientCert != "" || cfg.ClientKey != "" { + if err := b.AppendX509KeyPair(cfg.ClientCert, cfg.ClientKey); err != nil { return nil, fmt.Errorf("can't append key pair: %w", err) } } @@ -213,10 +197,6 @@ func newHTTPContext(ctx context.Context, conf httpContextCfg) (context.Context, return oidc.ClientContext(ctx, hc), nil } -func (c httpContextCfg) isZero() bool { - return c.rootCA == "" && c.caCert == "" && c.privateKey == "" && !c.sslSkipVerify -} - func hashToken(token, secret string) string { hash := sha512.New() _, _ = hash.Write([]byte(token + secret)) diff --git a/internal/app/config/loader/loader.go b/internal/app/config/loader/loader.go new file mode 100644 index 0000000..9aac91a --- /dev/null +++ b/internal/app/config/loader/loader.go @@ -0,0 +1,127 @@ +package loader + +import ( + "bytes" + "errors" + "fmt" + "os" + + "gopkg.in/yaml.v3" + + "github.com/ozontech/seq-ui/internal/app/config/migrate" + v1 "github.com/ozontech/seq-ui/internal/app/config/v1" + v2 "github.com/ozontech/seq-ui/internal/app/config/v2" +) + +const ( + V1 = 1 + V2 = 2 +) + +var ErrAlreadyLatestConfigVersion = errors.New("config is already at the latest schema version") + +type configMeta struct { + Version *int `yaml:"version"` +} + +// FromFile parse config from config path. +func FromFile(cfgPath string) (*v2.Config, error) { + cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec + if err != nil { + return nil, fmt.Errorf("read file: %w", err) + } + + version, err := readVersion(cfgBytes) + if err != nil { + return nil, fmt.Errorf("read version: %w", err) + } + + switch version { + case V1: + cfgV1, err := decode[v1.Config](cfgBytes, true) + if err != nil { + return nil, fmt.Errorf("parse config v1: %w", err) + } + cfgBytes, err = encode(migrate.V1ToV2(cfgV1)) + if err != nil { + return nil, err + } + case V2: + default: + return nil, fmt.Errorf("unsupported config version: %d", version) + } + + cfgBytes, err = mergeEnvOptions(cfgBytes) + if err != nil { + return nil, fmt.Errorf("merge env options: %w", err) + } + + cfg, err := decode[v2.Config](cfgBytes, true) + if err != nil { + return nil, fmt.Errorf("parse config v2: %w", err) + } + + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("validate config: %w", err) + } + + return &cfg, nil +} + +func ToLatestVersion(cfgBytes []byte) ([]byte, error) { + version, err := readVersion(cfgBytes) + if err != nil { + return nil, fmt.Errorf("read version: %w", err) + } + + switch version { + case V1: + cfgV1, err := decode[v1.Config](cfgBytes, true) + if err != nil { + return nil, fmt.Errorf("parse config v1: %w", err) + } + return encode(migrate.V1ToV2(cfgV1)) + case V2: + return nil, ErrAlreadyLatestConfigVersion + default: + return nil, fmt.Errorf("unsupported config version: %d", version) + } +} + +func readVersion(cfgBytes []byte) (int, error) { + meta, err := decode[configMeta](cfgBytes, false) + if err != nil { + return 0, err + } + + if meta.Version == nil { + return V1, nil + } + + return *meta.Version, nil +} + +func decode[T any](cfg []byte, strict bool) (T, error) { + var result T + + decoder := yaml.NewDecoder(bytes.NewReader(cfg)) + decoder.KnownFields(strict) + if err := decoder.Decode(&result); err != nil { + return result, err + } + + return result, nil +} + +func encode[T any](cfg T) ([]byte, error) { + var buf bytes.Buffer + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + if err := encoder.Encode(&cfg); err != nil { + return nil, fmt.Errorf("encode config v2: %w", err) + } + if err := encoder.Close(); err != nil { + return nil, fmt.Errorf("close encoder: %w", err) + } + return buf.Bytes(), nil +} diff --git a/internal/app/config/loader/merge.go b/internal/app/config/loader/merge.go new file mode 100644 index 0000000..0735768 --- /dev/null +++ b/internal/app/config/loader/merge.go @@ -0,0 +1,84 @@ +package loader + +import ( + "fmt" + "maps" + + "gopkg.in/yaml.v3" +) + +func mergeEnvOptions(cfgBytes []byte) ([]byte, error) { + var root map[string]any + if err := yaml.Unmarshal(cfgBytes, &root); err != nil { + return nil, fmt.Errorf("parse yaml for merge: %w", err) + } + + mergeHandlerOptions := func(root map[string]any) { + rootOpts, _ := root["options"].(map[string]any) + envs, _ := root["envs"].(map[string]any) + if rootOpts == nil || len(envs) == 0 { + return + } + + for _, e := range envs { + env, _ := e.(map[string]any) + if env == nil { + continue + } + + envOpts, _ := env["options"].(map[string]any) + env["options"] = mergeYAMLs(rootOpts, envOpts) + } + } + + mergeServerOptions := func(root map[string]any) { + rootOpts, _ := root["options"].(map[string]any) + envs, _ := root["envs"].(map[string]any) + if rootOpts == nil || len(envs) == 0 { + return + } + + for name, e := range envs { + envOpts, _ := e.(map[string]any) + envs[name] = mergeYAMLs(rootOpts, envOpts) + } + } + + handlers, _ := root["handlers"].(map[string]any) + for _, h := range handlers { + if handler, _ := h.(map[string]any); handler != nil { + mergeHandlerOptions(handler) + } + } + + server, _ := root["server"].(map[string]any) + if auth, _ := server["auth"].(map[string]any); auth != nil { + mergeServerOptions(auth) + } + if rl, _ := server["rate_limiting"].(map[string]any); rl != nil { + mergeServerOptions(rl) + } + out, err := yaml.Marshal(root) + if err != nil { + return nil, fmt.Errorf("marshal merged yaml: %w", err) + } + return out, nil +} + +func mergeYAMLs(rootOpts, envOpts map[string]any) map[string]any { + merged := make(map[string]any) + maps.Copy(merged, rootOpts) + + for k, v := range envOpts { + if existingValue, exists := merged[k]; exists { + if existingMap, ok := existingValue.(map[string]any); ok { + if newMap, ok := v.(map[string]any); ok { + merged[k] = mergeYAMLs(existingMap, newMap) + continue + } + } + } + merged[k] = v + } + return merged +} diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go new file mode 100644 index 0000000..22150ee --- /dev/null +++ b/internal/app/config/migrate/v1_v2.go @@ -0,0 +1,562 @@ +package migrate + +import ( + "time" + + v1 "github.com/ozontech/seq-ui/internal/app/config/v1" + v2 "github.com/ozontech/seq-ui/internal/app/config/v2" +) + +type v1tov2 struct { + src v1.Config +} + +func V1ToV2(src v1.Config) v2.Config { + m := v1tov2{src: src} + return m.migrate() +} + +func (m *v1tov2) migrate() v2.Config { + dst := v2.Config{Version: 2} + dst.Server = m.migrateServer(m.src.Server) + + var v1Cache v1.Cache + if m.src.Server != nil { + v1Cache = m.src.Server.Cache + } + + dst.Cache = m.migrateCache(v1Cache) + dst.Clients = m.migrateClients(m.src) + dst.DB = m.migrateDB(m.src.Server) + dst.Handlers = m.migrateHandlers(m.src.Handlers, v1Cache) + + return dst +} + +func (m *v1tov2) migrateServer(src *v1.Server) v2.Server { + if src == nil { + return v2.Server{} + } + + dst := v2.Server{ + HTTP: &v2.HTTP{ + Addr: src.HTTPAddr, + ReadTimeout: src.HTTPReadTimeout, + ReadHeaderTimeout: src.HTTPReadHeaderTimeout, + WriteTimeout: src.HTTPWriteTimeout, + CORS: m.migrateCORS(src.CORS), + }, + GRPC: &v2.GRPC{ + Addr: src.GRPCAddr, + ConnectionTimeout: src.GRPCConnectionTimeout, + }, + Debug: v2.Debug{ + Addr: src.DebugAddr, + ReadTimeout: src.HTTPReadTimeout, + ReadHeaderTimeout: src.HTTPReadHeaderTimeout, + WriteTimeout: src.HTTPWriteTimeout, + CORS: m.migrateCORS(src.CORS), + }, + RateLimiting: m.migrateRateLimiting(src.RateLimiters), + } + + if src.OIDC != nil { + oidc := &v2.OIDC{ + SkipVerify: src.OIDC.SkipVerify, + AuthURLs: src.OIDC.AuthURLs, + TLS: m.migrateTLS(src.OIDC), + AllowedClients: src.OIDC.AllowedClients, + } + + if src.OIDC.CacheSecretKey != "" { + cacheID, _ := m.defaultCacheIDs(src.Cache) + oidc.CacheID = cacheID + } + + dst.Auth = &v2.Auth{Options: v2.AuthOptions{OIDC: oidc}} + } + + if src.JWTSecretKey != "" { + if dst.Auth == nil { + dst.Auth = &v2.Auth{} + } + dst.Auth.Options.JWT = &v2.JWT{SecretKey: src.JWTSecretKey} + } + + return dst +} + +func (m *v1tov2) migrateTLS(src *v1.OIDC) *v2.TLS { + if src.RootCA == "" && src.CACert == "" && + src.PrivateKey == "" && !src.SSLSkipVerify { + return nil + } + + return &v2.TLS{ + CACert: src.RootCA, + ClientCert: src.CACert, + ClientKey: src.PrivateKey, + Insecure: src.SSLSkipVerify, + } +} + +func (m *v1tov2) migrateCORS(src *v1.CORS) *v2.CORS { + if src == nil { + return nil + } + + return &v2.CORS{ + AllowedOrigins: src.AllowedOrigins, + AllowedMethods: src.AllowedMethods, + AllowedHeaders: src.AllowedHeaders, + ExposedHeaders: src.ExposedHeaders, + AllowCredentials: src.AllowCredentials, + MaxAge: src.MaxAge, + OptionsPassthrough: src.OptionsPassthrough, + } +} + +func (m *v1tov2) migrateRateLimiting(src v1.ApiToRateLimiters) *v2.RateLimiting { + if len(src) == 0 { + return nil + } + + options := make(v2.ApiToRateLimiters, len(src)) + for api, rl := range src { + options[api] = v2.RateLimiters{ + Default: m.migrateRateLimiter(rl.Default), + SpecialUsers: m.migrateUserToRateLimiter(rl.SpecialUsers), + } + } + + return &v2.RateLimiting{Options: options} +} + +func (m *v1tov2) migrateUserToRateLimiter(src v1.UserToRateLimiter) map[string]v2.RateLimiter { + if src == nil { + return nil + } + + dst := make(map[string]v2.RateLimiter, len(src)) + for user, rl := range src { + dst[user] = m.migrateRateLimiter(rl) + } + + return dst +} + +func (m *v1tov2) migrateRateLimiter(src v1.RateLimiter) v2.RateLimiter { + return v2.RateLimiter{ + RatePerSec: src.RatePerSec, + MaxBurst: src.MaxBurst, + StoreMaxKeys: src.StoreMaxKeys, + PerHandler: src.PerHandler, + } +} + +func (m *v1tov2) migrateGRPCParams(src *v1.GRPCKeepaliveParams) *v2.GRPCParams { + if src == nil { + return nil + } + + return &v2.GRPCParams{ + Keepalive: &v2.KeepaliveParams{ + Time: src.Time, + Timeout: src.Timeout, + PermitWithoutStream: src.PermitWithoutStream, + }, + } +} + +func (m *v1tov2) migrateClients(src v1.Config) v2.Clients { + dst := v2.Clients{} + + if src.Clients != nil { + if len(src.Clients.SeqDB) > 0 { + dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) + for i := range src.Clients.SeqDB { + dst.SeqDB = append(dst.SeqDB, m.migrateSeqDBClient(&src.Clients.SeqDB[i])) + } + } else { + dst.SeqDB = []v2.SeqDBClient{{ + ID: v2.DefaultSeqDBClientID, + Timeout: src.Clients.SeqDBTimeout, + AvgDocSize: src.Clients.SeqDBAvgDocSize, + Addrs: src.Clients.SeqDBAddrs, + RequestRetries: src.Clients.RequestRetries, + RequestRetriesBackoff: m.migrateBackoff(src.Clients.InitialRetryBackoff, src.Clients.MaxRetryBackoff), + ClientMode: src.Clients.ProxyClientMode, + GRPCParams: m.migrateGRPCParams(src.Clients.GRPCKeepaliveParams), + }} + } + } + + if src.Server != nil && src.Server.CH != nil { + ch := src.Server.CH + dst.ClickHouse = []v2.CHClient{{ + ID: v2.DefaultCHClientID, + Addrs: ch.Addrs, + Database: ch.Database, + Username: ch.Username, + Password: ch.Password, + Sharded: ch.Sharded, + DialTimeout: ch.DialTimeout, + ReadTimeout: ch.ReadTimeout, + }} + } + + return dst +} + +func (m *v1tov2) migrateBackoff(initialInterval, maxInterval time.Duration) *v2.Backoff { + if initialInterval == 0 && maxInterval == 0 { + return nil + } + + return &v2.Backoff{ + InitialInterval: initialInterval, + MaxInterval: maxInterval, + } +} + +func (m *v1tov2) migrateSeqDBClient(src *v1.SeqDBClient) v2.SeqDBClient { + return v2.SeqDBClient{ + ID: src.ID, + Timeout: src.Timeout, + AvgDocSize: src.AvgDocSize, + Addrs: src.Addrs, + RequestRetries: src.RequestRetries, + RequestRetriesBackoff: m.migrateBackoff(src.InitialRetryBackoff, src.MaxRetryBackoff), + ClientMode: src.ClientMode, + GRPCParams: m.migrateGRPCParams(src.GRPCKeepaliveParams), + } +} + +func (m *v1tov2) migrateDB(src *v1.Server) *v2.DB { + if src == nil || src.DB == nil { + return nil + } + + return &v2.DB{ + Name: src.DB.Name, + Host: src.DB.Host, + Port: src.DB.Port, + User: src.DB.User, + Pass: src.DB.Pass, + RequestTimeout: src.DB.RequestTimeout, + ConnectionPoolCapacity: src.DB.ConnectionPoolCapacity, + UsePreparedStatements: src.DB.UsePreparedStatements, + } +} + +func (m *v1tov2) migrateCache(src v1.Cache) v2.Cache { + // in config v1 inmemory cache is always created + cache := v2.Cache{ + Inmemory: []v2.Inmemory{{ + ID: v2.DefaultInmemID, + NumCounters: src.Inmemory.NumCounters, + MaxCost: src.Inmemory.MaxCost, + BufferItems: src.Inmemory.BufferItems, + }}, + } + + if src.Redis != nil { + cache.Redis = append(cache.Redis, + m.migrateRedis(src.Redis, v2.DefaultRedisWithInmemID, v2.DefaultInmemID), + m.migrateRedis(src.Redis, v2.DefaultRedisWithoutInmemID, ""), + ) + } + + return cache +} + +func (m *v1tov2) migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { + return v2.Redis{ + ID: id, + WithInmemID: withInmemID, + Addr: src.Addr, + Username: src.Username, + Password: src.Password, + Timeout: src.Timeout, + MaxRetries: src.MaxRetries, + MinRetryBackoff: src.MinRetryBackoff, + MaxRetryBackoff: src.MaxRetryBackoff, + KeyPrefix: src.KeyPrefix, + } +} + +func (m *v1tov2) migrateHandlers(src *v1.Handlers, v1Cache v1.Cache) v2.Handlers { + if src == nil { + src = &v1.Handlers{} + } + + dst := v2.Handlers{ + SeqAPI: m.migrateSeqAPI(&src.SeqAPI, v1Cache), + AsyncSearch: m.migrateAsyncSearch(&src.AsyncSearch, &src.SeqAPI), + ErrorGroups: m.migrateErrorGroups(&src.ErrorGroups, m.src.Server), + Admin: m.migrateAdmin(src.Admin, v1Cache), + // MassExport: m.migrateMassExport(src.MassExport, cfg), + } + + return dst +} + +func (m *v1tov2) migrateSeqAPI(src *v1.SeqAPI, v1Cache v1.Cache) *v2.SeqAPI { + dst := &v2.SeqAPI{ + Options: m.migrateSeqAPIOptions(src.SeqAPIOptions, v1Cache), + DefaultEnv: src.DefaultEnv, + } + + if len(src.Envs) > 0 { + dst.Envs = make(map[string]v2.SeqAPIEnv, len(src.Envs)) + for envName, envCfg := range src.Envs { + dstEnv := v2.SeqAPIEnv{SeqDBID: envCfg.SeqDB} + if envCfg.Options != nil { + options := m.migrateSeqAPIOptions(envCfg.Options, v1Cache) + dstEnv.Options = &options + } + dst.Envs[envName] = dstEnv + } + } + + return dst +} + +func (m *v1tov2) migrateSeqAPIOptions(src *v1.SeqAPIOptions, v1Cache v1.Cache) v2.SeqAPIOptions { + if src == nil { + src = &v1.SeqAPIOptions{} + } + + redisWithInmemID, redisOrInmemID := m.defaultCacheIDs(v1Cache) + dst := v2.SeqAPIOptions{ + Limits: v2.SeqAPILimits{ + MaxSearchLimit: src.MaxSearchLimit, + MaxSearchTotal: src.MaxSearchTotalLimit, + MaxSearchOffset: src.MaxSearchOffsetLimit, + MaxExportLimit: src.MaxExportLimit, + SeqCLIMaxSearchLimit: src.SeqCLIMaxSearchLimit, + MaxParallelExportRequests: src.MaxParallelExportRequests, + MaxAggregationsPerRequest: src.MaxAggregationsPerRequest, + MaxBucketsPerAggregationTs: src.MaxBucketsPerAggregationTs, + }, + Masking: m.migrateMasking(src.Masking), + PinnedFields: m.migrateFields(src.PinnedFields), + SystemFields: m.migrateFields(src.SystemFields), + Caches: v2.SeqAPICaches{ + Events: &v2.HandlerCache{ + ID: redisWithInmemID, + TTL: src.EventsCacheTTL, + }, + LogsLifespan: &v2.HandlerCache{ + ID: redisOrInmemID, + TTL: src.LogsLifespanCacheTTL, + }, + }, + } + + if src.FieldsCacheTTL > 0 { + dst.Caches.Fields = &v2.HandlerCache{ + ID: v2.DefaultInmemID, + TTL: src.FieldsCacheTTL, + } + } + + return dst +} + +func (m *v1tov2) migrateFields(fs []v1.Field) []v2.Field { + if fs == nil { + return nil + } + + dst := make([]v2.Field, len(fs)) + for i, f := range fs { + dst[i] = v2.Field{ + Name: f.Name, + Type: f.Type, + } + } + + return dst +} + +func (m *v1tov2) migrateMasking(src *v1.Masking) *v2.Masking { + if src == nil { + return nil + } + + return &v2.Masking{ + Masks: m.migrateMasks(src.Masks), + ProcessFields: src.ProcessFields, + IgnoreFields: src.IgnoreFields, + } +} + +func (m *v1tov2) migrateMasks(ms []v1.Mask) []v2.Mask { + if ms == nil { + return nil + } + + dst := make([]v2.Mask, len(ms)) + for i := range ms { + mask := &ms[i] + dst[i] = v2.Mask{ + Re: mask.Re, + Groups: mask.Groups, + Mode: mask.Mode, + ReplaceWord: mask.ReplaceWord, + ProcessFields: mask.ProcessFields, + IgnoreFields: mask.IgnoreFields, + FieldFilters: m.migrateFieldFilters(mask.FieldFilters), + } + } + + return dst +} + +func (m *v1tov2) migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { + if src == nil { + return nil + } + + dst := &v2.FieldFilterSet{ + Condition: src.Condition, + } + if src.Filters != nil { + dst.Filters = make([]v2.FieldFilter, len(src.Filters)) + for i, f := range src.Filters { + dst.Filters[i] = v2.FieldFilter{ + Field: f.Field, + Mode: f.Mode, + Values: f.Values, + } + } + } + + return dst +} + +func (m *v1tov2) migrateErrorGroups(eg *v1.ErrorGroups, server *v1.Server) *v2.ErrorGroups { + if server == nil || server.CH == nil { + return nil + } + + return &v2.ErrorGroups{ + Options: v2.ErrorGroupsOptions{ + LogTagsMapping: v2.LogTagsMapping{ + Env: eg.LogTagsMapping.Env, + Service: eg.LogTagsMapping.Service, + Release: eg.LogTagsMapping.Release, + }, + QueryFilter: eg.QueryFilter, + }, + } +} + +func (m *v1tov2) migrateAsyncSearch(src *v1.AsyncSearch, seqAPI *v1.SeqAPI) *v2.AsyncSearch { + dst := &v2.AsyncSearch{ + Options: v2.AsyncSearchOptions{ + AdminUsers: src.AdminUsers, + ListQueryLengthLimit: src.ListQueryLengthLimit, + }, + DefaultEnv: seqAPI.DefaultEnv, + } + + if len(seqAPI.Envs) > 0 { + dst.Envs = make(map[string]v2.AsyncSearchEnv, len(seqAPI.Envs)) + for envName, envCfg := range seqAPI.Envs { + dst.Envs[envName] = v2.AsyncSearchEnv{SeqDBID: envCfg.SeqDB} + } + } + + return dst +} + +// func (m *v1tov2) migrateMassExport(src *v1.MassExport, cfg *v2.Config) *v2.MassExport { +// if src == nil { +// return nil +// } +// +// dst := &v2.MassExport{ +// SeqDBID: v2.DefaultSeqDBClientID, +// Options: v2.MassExportOptions{ +// BatchSize: src.BatchSize, +// WorkersCount: src.WorkersCount, +// TasksChannelSize: src.TasksChannelSize, +// PartLength: src.PartLength, +// URLPrefix: src.URLPrefix, +// AllowedUsers: src.AllowedUsers, +// FileStore: m.migrateFileStore(src.FileStore), +// DownloadParams: m.migrateDownloadParams(m.src.SeqProxyDownloader), +// }, +// } +// +// if me.SessionStore != nil { +// cfg.Cache.Redis = append(cfg.Cache.Redis, m.migrateRedis(&src.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) +// +// dst.Options.SessionStore = v2.SessionStore{ +// RedisID: v2.DefaultMassExportRedisID, +// ExportLifetime: src.SessionStore.ExportLifetime, +// } +// } +// +// return dst +// } +// +// func (m *v1tov2) migrateFileStore(fs *v1.FileStore) v2.FileStore { +// if fs == nil || fs.S3 == nil { +// return v2.FileStore{} +// } +// +// return v2.FileStore{ +// S3: &v2.S3{ +// Endpoint: fs.S3.Endpoint, +// AccessKeyID: fs.S3.AccessKeyID, +// SecretAccessKey: fs.S3.SecretAccessKey, +// BucketName: fs.S3.BucketName, +// EnableSSL: fs.S3.EnableSSl, +// }, +// } +// } + +// func (m *v1tov2) migrateDownloadParams(src *v1.SeqProxyDownloader) v2.DownloadParams { +// if src == nil { +// return v2.DownloadParams{} +// } +// +// return v2.DownloadParams{ +// Delay: src.Delay, +// InitialRetryBackoff: src.InitialRetryBackoff, +// MaxRetryBackoff: src.MaxRetryBackoff, +// } +// } +func (m *v1tov2) migrateAdmin(src *v1.Admin, v1Cache v1.Cache) *v2.Admin { + if src == nil { + return nil + } + + dst := &v2.Admin{ + Options: v2.AdminOptions{ + SuperUsers: src.SuperUsers, + }, + } + + if v1Cache.Redis != nil { + dst.Options.Cache = &v2.HandlerCache{ + ID: v2.DefaultRedisWithoutInmemID, + TTL: src.CacheTTL, + } + } + + return dst +} + +// defaultCacheIDs returns cache ids same as in v1. +// inmemWithRedisID - inmemory+redis or inmemory (v1 events, oidc) +// redisOrInmemID - redis or inmemory (v1 logslifespan) +func (m *v1tov2) defaultCacheIDs(src v1.Cache) (inmemWithRedisID, redisOrInmemID string) { + if src.Redis != nil { + return v2.DefaultRedisWithInmemID, v2.DefaultRedisWithoutInmemID + } + return v2.DefaultInmemID, v2.DefaultInmemID +} diff --git a/internal/app/config/migrate/v1_v2_test.go b/internal/app/config/migrate/v1_v2_test.go new file mode 100644 index 0000000..906eaec --- /dev/null +++ b/internal/app/config/migrate/v1_v2_test.go @@ -0,0 +1,3 @@ +package migrate + +// TODO: write tests diff --git a/internal/app/config/config.go b/internal/app/config/v1/config.go similarity index 60% rename from internal/app/config/config.go rename to internal/app/config/v1/config.go index cf926f9..929d81e 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/v1/config.go @@ -1,59 +1,225 @@ package config import ( - "bytes" - "fmt" - "os" "time" - - "gopkg.in/yaml.v3" ) -const ( - DefaultSeqDBClientID = "default" - - ProxyClientModeGRPC = "grpc" - - MaskModeMask = "mask" - MaskModeReplace = "replace" - MaskModeCut = "cut" - - FieldFilterConditionAnd = "and" - FieldFilterConditionOr = "or" - FieldFilterConditionNot = "not" - - FieldFilterModeEqual = "equal" - FieldFilterModeContains = "contains" - FieldFilterModePrefix = "prefix" - FieldFilterModeSuffix = "suffix" - - minGRPCKeepaliveTime = 10 * time.Second - minGRPCKeepaliveTimeout = 1 * time.Second - - defaultAsyncSearchListQueryLengthLimit = 1000 - - defaultMaxSearchTotalLimit = 1000000 - defaultMaxSearchOffsetLimit = 1000000 - defaultMaxExportLimit = 100000 - defaultMaxAggregationsPerRequest = 1 - defaultMaxBucketsPerAggregationTs = 200 - defaultMaxParallelExportRequests = 1 - - defaultInmemCacheNumCounters = 10000000 - defaultInmemCacheMaxCost = 1000000 - defaultInmemCacheBufferItems = 64 - - defaultEventsCacheTTL = 24 * time.Hour - - defaultLogsLifespanCacheKey = "logs_lifespan" - defaultLogsLifespanCacheTTL = 10 * time.Minute - defaultAdminCacheTTL = time.Hour - - defaultClickHouseDialTimeout = 5 * time.Second - defaultClickHouseReadTimeout = 30 * time.Second -) +// Configuration scheme +// server: +// http_addr: +// grpc_addr: +// debug_addr: +// grpc_connection_timeout: +// http_read_timeout: +// http_read_header_timeout: +// http_write_timeout: +// cors: +// allowed_origins: +// allowed_methods: +// allowed_headers: +// exposed_headers: +// allow_credentials: +// max_age: +// options_passthrough: +// jwt_secret_key: +// oidc: +// skip_verify: +// auth_urls: +// root_ca: +// ca_cert: +// private_key: +// ssl_skip_verify: +// allowed_clients: +// cache_secret_key: +// rate_limiters: +// : +// default: +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// spec_users: +// : +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// cache: +// inmemory: +// num_counters: +// max_cost: +// buffer_items: +// redis: +// key_prefix: +// addr: +// username: +// password: +// timeout: +// max_retries: +// min_retry_backoff: +// max_retry_backoff: +// db: +// name: +// host: +// port: +// pass: +// user: +// request_timeout: +// connection_pool_capacity: +// use_prepared_statements: +// clickhouse: +// addrs: +// database: +// username: +// password: +// sharded: +// dial_timeout: +// read_timeout: +// clients: +// seq_db_timeout: +// seq_db_avg_doc_size: +// seq_db_addrs: +// request_retries: +// initial_retry_backoff: +// max_retry_backoff: +// proxy_client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: +// seq_db: +// - id: +// timeout: +// avg_doc_size: +// addrs: +// request_retries: +// initial_retry_backoff: +// max_retry_backoff: +// client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: +// handlers: +// seq_api: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// - name: +// type: +// system_fields: +// - name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// - re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// - field: +// mode: +// values: +// process_fields: +// ignore_fields: +// envs: +// : +// seq_db_id: +// options: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// - name: +// type: +// system_fields: +// - name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// - re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// - field: +// mode: +// values: +// process_fields: +// ignore_fields: +// default_env: +// error_groups: +// log_tags_mapping: +// env: +// service: +// release: +// query_filter: +// : +// mass_export: +// batch_size: +// workers_count: +// tasks_channel_size: +// part_length: +// url_prefix: +// allowed_users: +// file_store: +// s3: +// endpoint: +// access_key_id: +// secret_access_key: +// bucket_name: +// enable_ssl: +// session_store: +// redis: +// key_prefix +// addr: +// username: +// password: +// timeout: +// max_retries: +// min_retry_backoff: +// max_retry_backoff: +// export_lifetime: +// seq_proxy_downloader: +// delay: +// initial_retry_backoff: +// max_retry_backoff: +// async_search: +// admin_users: +// list_query_length_limit: +// admin: +// super_users: +// cache_ttl: type Config struct { + Version *int `yaml:"version"` Server *Server `yaml:"server"` Clients *Clients `yaml:"clients"` Handlers *Handlers `yaml:"handlers"` @@ -91,10 +257,6 @@ type DB struct { UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` } -func (db *DB) ConnString() string { - return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) -} - type CH struct { Addrs []string `yaml:"addrs"` Database string `yaml:"database"` @@ -130,6 +292,7 @@ type InmemoryCache struct { } type Redis struct { + KeyPrefix string `yaml:"key_prefix"` Addr string `yaml:"addr"` Username string `yaml:"username"` Password string `yaml:"password"` @@ -137,7 +300,6 @@ type Redis struct { MaxRetries int `yaml:"max_retries"` MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - KeyPrefix string `yaml:"key_prefix"` } type Cache struct { @@ -327,164 +489,3 @@ type Admin struct { SuperUsers []string `yaml:"super_users"` CacheTTL time.Duration `yaml:"cache_ttl"` } - -// FromFile parse config from config path. -func FromFile(cfgPath string) (Config, error) { - cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec - if err != nil { - return Config{}, fmt.Errorf("error reading file: %s", err) - } - - cfg, err := parse(cfgBytes) - if err != nil { - return Config{}, fmt.Errorf("error parsing file: %s", err) - } - - proxyClientMode := cfg.Clients.ProxyClientMode - if proxyClientMode == "" { - cfg.Clients.ProxyClientMode = ProxyClientModeGRPC - } else if proxyClientMode != ProxyClientModeGRPC { - return Config{}, fmt.Errorf( - "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", - proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, - ) - } - - setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) - - if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { - cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit - } - - if cfg.Server.DB != nil && cfg.Server.DB.UsePreparedStatements == nil { - cfg.Server.DB.UsePreparedStatements = new(bool) - *cfg.Server.DB.UsePreparedStatements = true - } - - if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { - cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout - } - - if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { - cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout - } - - if cfg.Clients.GRPCKeepaliveParams != nil { - if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { - cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime - } - if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { - cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout - } - } - - if cfg.Server.Cache.Inmemory.NumCounters <= 0 { - cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters - } - if cfg.Server.Cache.Inmemory.MaxCost <= 0 { - cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost - } - if cfg.Server.Cache.Inmemory.BufferItems <= 0 { - cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems - } - - if cfg.Handlers.Admin != nil { - if cfg.Handlers.Admin.CacheTTL <= 0 { - cfg.Handlers.Admin.CacheTTL = defaultAdminCacheTTL - } - } - - if len(cfg.Clients.SeqDB) == 0 { - defaultClient := SeqDBClient{ - ID: DefaultSeqDBClientID, - Timeout: cfg.Clients.SeqDBTimeout, - AvgDocSize: cfg.Clients.SeqDBAvgDocSize, - Addrs: cfg.Clients.SeqDBAddrs, - RequestRetries: cfg.Clients.RequestRetries, - InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, - MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, - ClientMode: cfg.Clients.ProxyClientMode, - GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, - } - cfg.Clients.SeqDB = []SeqDBClient{defaultClient} - } - - clientIDs := make(map[string]struct{}) - for _, client := range cfg.Clients.SeqDB { - if client.ID == "" { - return Config{}, fmt.Errorf("seq_db client ID cannot be empty") - } - if _, ok := clientIDs[client.ID]; ok { - return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) - } - clientIDs[client.ID] = struct{}{} - } - - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return Config{}, fmt.Errorf("default_env must be specified when using envs") - } - - if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) - } - - for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if _, ok := clientIDs[envConfig.SeqDB]; !ok { - return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) - } - - if envConfig.Options == nil { - envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions - } else { - setSeqAPIOptionsDefaults(envConfig.Options) - } - - cfg.Handlers.SeqAPI.Envs[envName] = envConfig - } - } - - return cfg, nil -} - -func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { - if options.MaxAggregationsPerRequest <= 0 { - options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest - } - if options.MaxBucketsPerAggregationTs <= 0 { - options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs - } - if options.MaxParallelExportRequests <= 0 { - options.MaxParallelExportRequests = defaultMaxParallelExportRequests - } - if options.MaxSearchTotalLimit <= 0 { - options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit - } - if options.MaxSearchOffsetLimit <= 0 { - options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit - } - if options.MaxExportLimit <= 0 { - options.MaxExportLimit = defaultMaxExportLimit - } - if options.EventsCacheTTL <= 0 { - options.EventsCacheTTL = defaultEventsCacheTTL - } - if options.LogsLifespanCacheKey == "" { - options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey - } - if options.LogsLifespanCacheTTL <= 0 { - options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL - } -} - -func parse(cfg []byte) (Config, error) { - result := Config{} - - decoder := yaml.NewDecoder(bytes.NewReader(cfg)) - decoder.KnownFields(true) - if err := decoder.Decode(&result); err != nil { - return result, fmt.Errorf("error parsing config: %w", err) - } - - return result, nil -} diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go new file mode 100644 index 0000000..9a26415 --- /dev/null +++ b/internal/app/config/v2/config.go @@ -0,0 +1,638 @@ +package config + +import ( + "fmt" + "slices" + "time" +) + +// Configuration scheme +// version: +// server: +// http: +// addr: +// read_timeout: +// read_header_timeout: +// write_timeout: +// cors: +// allowed_origins: +// allowed_methods: +// allowed_headers: +// exposed_headers: +// allow_credentials: +// max_age: +// options_passthrough: +// grpc: +// addr: +// connection_timeout: +// debug: +// addr: +// read_timeout: +// read_header_timeout: +// write_timeout: +// cors: +// allowed_origins: +// allowed_methods: +// allowed_headers: +// exposed_headers: +// allow_credentials: +// max_age: +// options_passthrough: +// auth: +// options: +// oidc: +// cache_id: +// cache_secret_key: +// skip_verify: +// auth_urls: +// tls: +// ca_cert: +// client_cert: +// client_key: +// insecure: +// allowed_clients: +// jwt: +// secret_key: +// envs: +// : # same as root options +// rate_limiting: +// options: +// : +// default: +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// spec_users: +// : +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// envs: +// : # same as root options +// clients: +// seq_db: +// - id: +// timeout: +// avg_doc_size: +// addrs: +// request_retries_backoff: +// backoff: +// initial_interval: +// max_interval +// client_mode: +// grpc_params: +// keepalive: +// time: +// timeout: +// permit_without_stream: +// clickhouse: +// - id: +// addrs: +// database: +// username: +// password: +// sharded: +// dial_timeout: +// read_timeout: +// handlers: +// seq_api: +// options: +// limits: +// max_search_limit: +// max_search_total: +// max_search_offset: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// masking: +// masks: +// - re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// - field: +// mode: +// values: +// process_fields: +// ignore_fields: +// caches: +// events: +// id: +// ttl: +// logs_lifespan: +// id: +// ttl: +// fields: +// id: +// ttl: +// pinned_fields: +// - name: +// type: +// system_fields: +// - name: +// type: +// envs: +// : +// seq_db_id: +// options: # same as root options +// default_env: +// error_groups: +// options: +// log_tags_mapping: +// env: +// service: +// release: +// query_filter: +// : +// envs: +// : +// ch_id: +// options: # same as root options +// default_env: +// mass_export: +// options: +// workers_count: +// tasks_channel_size: +// part_length: +// url_prefix: +// allowed_users: +// file_store: +// s3: +// endpoint: +// access_key_id: +// secret_access_key: +// bucket_name: +// enable_ssl: +// session_store: +// lifetime: +// cache_id: +// download_params: +// batch_size: +// delay: +// delay_backoff: +// initial_interval: +// max_interval: +// envs: +// : +// seq_db_id: +// options: # same as root options +// default_env: +// async_search: +// options: +// admin_users: +// list_query_length_limit: +// envs: +// : +// seq_db_id: +// options: # same as root options +// default_env: +// admin: +// options: +// super_users: +// cache: +// id: +// ttl: +// envs: +// : +// options: # same as root options +// default_env: +// db: +// name: +// host: +// port: +// user: +// pass: +// request_timeout: +// connection_pool_capacity: +// use_prepared_statements: +// cache: +// inmemory: +// - id: +// num_counters: +// max_cost: +// buffer_items: +// redis: +// - id: +// with_inmem_id: +// key_prefix: +// addr: +// username: +// password: +// max_retries: +// timeout: +// min_retry_backoff: +// max_retry_backoff: + +type Config struct { + Version int `yaml:"version"` + Server Server `yaml:"server"` + Clients Clients `yaml:"clients"` + Handlers Handlers `yaml:"handlers"` + Cache Cache `yaml:"cache"` + DB *DB `yaml:"db"` +} + +type CORS struct { + AllowedOrigins []string `yaml:"allowed_origins"` + AllowedMethods []string `yaml:"allowed_methods"` + AllowedHeaders []string `yaml:"allowed_headers"` + ExposedHeaders []string `yaml:"exposed_headers"` + MaxAge int `yaml:"max_age"` + AllowCredentials bool `yaml:"allow_credentials"` + OptionsPassthrough bool `yaml:"options_passthrough"` +} + +type TLS struct { + CACert string `yaml:"ca_cert"` + ClientCert string `yaml:"client_cert"` + ClientKey string `yaml:"client_key"` + Insecure bool `yaml:"insecure"` +} + +type OIDC struct { + CacheID string `yaml:"cache_id"` + CacheSecretKey string `yaml:"cache_secret_key"` + SkipVerify bool `yaml:"skip_verify"` + AuthURLs []string `yaml:"auth_urls"` + AllowedClients []string `yaml:"allowed_clients"` + TLS *TLS `yaml:"tls"` +} + +type DB struct { + Name string `yaml:"name"` + Host string `yaml:"host"` + Port int64 `yaml:"port"` + User string `yaml:"user"` + Pass string `yaml:"pass"` + RequestTimeout time.Duration `yaml:"request_timeout"` + ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` + UsePreparedStatements *bool `yaml:"use_prepared_statements"` +} + +func (db *DB) ConnString() string { + return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) +} + +type ( + RateLimiter struct { + RatePerSec int `yaml:"rate_per_sec"` + MaxBurst int `yaml:"max_burst"` + StoreMaxKeys int `yaml:"store_max_keys"` + PerHandler bool `yaml:"per_handler"` + } + + RateLimiters struct { + Default RateLimiter `yaml:"default"` + SpecialUsers map[string]RateLimiter `yaml:"spec_users"` + } + + ApiToRateLimiters map[string]RateLimiters +) + +type Inmemory struct { + ID string `yaml:"id"` + NumCounters int64 `yaml:"num_counters"` + MaxCost int64 `yaml:"max_cost"` + BufferItems int64 `yaml:"buffer_items"` +} + +type Redis struct { + ID string `yaml:"id"` + WithInmemID string `yaml:"with_inmem_id"` + KeyPrefix string `yaml:"key_prefix"` + Addr string `yaml:"addr"` + Username string `yaml:"username"` + Password string `yaml:"password"` + MaxRetries int `yaml:"max_retries"` + Timeout time.Duration `yaml:"timeout"` + MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` +} + +type Cache struct { + Inmemory []Inmemory `yaml:"inmemory"` + Redis []Redis `yaml:"redis"` +} + +func (c *Cache) InmemByID(id string) *Inmemory { + i := slices.IndexFunc(c.Inmemory, func(inm Inmemory) bool { + return inm.ID == id + }) + if i == -1 { + return nil + } + + return &c.Inmemory[i] +} + +func (c *Cache) RedisByID(id string) *Redis { + i := slices.IndexFunc(c.Redis, func(r Redis) bool { + return r.ID == id + }) + if i == -1 { + return nil + } + + return &c.Redis[i] +} + +type S3 struct { + Endpoint string `yaml:"endpoint"` + AccessKeyID string `yaml:"access_key_id"` + SecretAccessKey string `yaml:"secret_access_key"` + BucketName string `yaml:"bucket_name"` + EnableSSL bool `yaml:"enable_ssl"` +} + +type DownloadParams struct { + BatchSize uint64 `yaml:"batch_size"` + Delay time.Duration `yaml:"delay"` + DelayBackoff Backoff `yaml:"delay_backoff"` +} + +type SessionStore struct { + Lifetime time.Duration `yaml:"lifetime"` + CacheID string `yaml:"cache_id"` +} + +type FileStore struct { + S3 *S3 `yaml:"s3"` +} + +type MassExport struct { + Options MassExportOptions `yaml:"options"` + Envs map[string]MassExportEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type MassExportEnv struct { + SeqDBID string `yaml:"seq_db_id"` + Options *MassExportOptions `yaml:"options"` +} + +type MassExportOptions struct { + WorkersCount int `yaml:"workers_count"` + TasksChannelSize int `yaml:"tasks_channel_size"` + PartLength time.Duration `yaml:"part_length"` + URLPrefix string `yaml:"url_prefix"` + AllowedUsers []string `yaml:"allowed_users"` + FileStore FileStore `yaml:"file_store"` + SessionStore SessionStore `yaml:"session_store"` + DownloadParams DownloadParams `yaml:"download_params"` +} + +type HTTP struct { + Addr string `yaml:"addr"` + ReadHeaderTimeout time.Duration `yaml:"read_header_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` + WriteTimeout time.Duration `yaml:"write_timeout"` + CORS *CORS `yaml:"cors"` +} + +type GRPC struct { + Addr string `yaml:"addr"` + ConnectionTimeout time.Duration `yaml:"connection_timeout"` +} + +type Debug struct { + Addr string `yaml:"addr"` + ReadHeaderTimeout time.Duration `yaml:"read_header_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` + WriteTimeout time.Duration `yaml:"write_timeout"` + CORS *CORS `yaml:"cors"` +} + +type JWT struct { + SecretKey string `yaml:"secret_key"` +} + +type Auth struct { + Options AuthOptions `yaml:"options"` + Envs map[string]AuthOptions `yaml:"envs"` +} + +type AuthOptions struct { + OIDC *OIDC `yaml:"oidc"` + JWT *JWT `yaml:"jwt"` +} + +type Server struct { + HTTP *HTTP `yaml:"http"` + GRPC *GRPC `yaml:"grpc"` + Debug Debug `yaml:"debug"` + Auth *Auth `yaml:"auth"` + RateLimiting *RateLimiting `yaml:"rate_limiting"` +} + +type RateLimiting struct { + Options ApiToRateLimiters `yaml:"options"` + Envs map[string]ApiToRateLimiters `yaml:"envs"` +} + +type KeepaliveParams struct { + // After a duration of this time if the client doesn't see any activity it + // pings the server to see if the transport is still alive. + // If set below 10s, a minimum value of 10s will be used instead. + Time time.Duration `yaml:"time"` + // After having pinged for keepalive check, the client waits for a duration + // of Timeout and if no activity is seen even after that the connection is + // closed. If set below 1s, a minimum value of 1s will be used instead. + Timeout time.Duration `yaml:"timeout"` + // If true, client sends keepalive pings even with no active RPCs. If false, + // when there are no active RPCs, Time and Timeout will be ignored and no + // keepalive pings will be sent. False by default. + PermitWithoutStream bool `yaml:"permit_without_stream"` +} + +type GRPCParams struct { + Keepalive *KeepaliveParams `yaml:"keepalive"` +} + +type SeqDBClient struct { + ID string `yaml:"id"` + Timeout time.Duration `yaml:"timeout"` + AvgDocSize int `yaml:"avg_doc_size"` + Addrs []string `yaml:"addrs"` + RequestRetries int `yaml:"request_retries"` + RequestRetriesBackoff *Backoff `yaml:"request_retries_backoff"` + ClientMode string `yaml:"client_mode"` + GRPCParams *GRPCParams `yaml:"grpc_params"` +} + +type Backoff struct { + InitialInterval time.Duration `yaml:"initial_interval"` + MaxInterval time.Duration `yaml:"max_interval"` +} + +type CHClient struct { + ID string `yaml:"id"` + Addrs []string `yaml:"addrs"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Sharded bool `yaml:"sharded"` + DialTimeout time.Duration `yaml:"dial_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` +} + +type Clients struct { + SeqDB []SeqDBClient `yaml:"seq_db"` + ClickHouse []CHClient `yaml:"clickhouse"` +} + +func (c *Clients) ClickHouseByID(id string) *CHClient { + i := slices.IndexFunc(c.ClickHouse, func(ch CHClient) bool { + return ch.ID == id + }) + if i == -1 { + return nil + } + + return &c.ClickHouse[i] +} + +type Handlers struct { + SeqAPI *SeqAPI `yaml:"seq_api"` + MassExport *MassExport `yaml:"mass_export"` + AsyncSearch *AsyncSearch `yaml:"async_search"` + ErrorGroups *ErrorGroups `yaml:"error_groups"` + Admin *Admin `yaml:"admin"` +} + +type Admin struct { + Options AdminOptions `yaml:"options"` + Envs map[string]AdminEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type AdminEnv struct { + Options *AdminOptions `yaml:"options"` +} + +type AdminOptions struct { + SuperUsers []string `yaml:"super_users"` + Cache *HandlerCache `yaml:"cache"` +} + +type Field struct { + Name string `yaml:"name"` + Type string `yaml:"type"` +} + +type SeqAPI struct { + // добавить проверку на len(SEQDB) == 1 и берем [0] + Options SeqAPIOptions `yaml:"options"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type SeqAPIEnv struct { + SeqDBID string `yaml:"seq_db_id"` + Options *SeqAPIOptions `yaml:"options"` +} + +type SeqAPIOptions struct { + Limits SeqAPILimits `yaml:"limits"` + Caches SeqAPICaches `yaml:"caches"` + Masking *Masking `yaml:"masking"` + PinnedFields []Field `yaml:"pinned_fields"` + SystemFields []Field `yaml:"system_fields"` +} + +type SeqAPILimits struct { + MaxSearchLimit int32 `yaml:"max_search_limit"` + MaxSearchTotal int64 `yaml:"max_search_total"` + MaxSearchOffset int32 `yaml:"max_search_offset"` + MaxExportLimit int32 `yaml:"max_export_limit"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` +} + +type SeqAPICaches struct { + Events *HandlerCache `yaml:"events"` + LogsLifespan *HandlerCache `yaml:"logs_lifespan"` + Fields *HandlerCache `yaml:"fields"` +} + +type HandlerCache struct { + ID string `yaml:"id"` + TTL time.Duration `yaml:"ttl"` +} + +type Masking struct { + Masks []Mask `yaml:"masks"` + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` +} + +type Mask struct { + Re string `yaml:"re"` + Groups []int `yaml:"groups"` + Mode string `yaml:"mode"` // "mask" or "replace" or "cut" + ReplaceWord string `yaml:"replace_word"` // for mode:replace + + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` + + FieldFilters *FieldFilterSet `yaml:"field_filters"` +} + +type FieldFilter struct { + Field string `yaml:"field"` + Mode string `yaml:"mode"` // "equal" or "contains" or "prefix" or "suffix" + Values []string `yaml:"values"` +} + +type FieldFilterSet struct { + Condition string `yaml:"condition"` // "and" or "or" or "not" + Filters []FieldFilter `yaml:"filters"` // max 1 if condition:not +} + +type LogTagsMapping struct { + Release []string `yaml:"release"` + Service []string `yaml:"service"` + Env []string `yaml:"env"` +} + +type ErrorGroups struct { + Options ErrorGroupsOptions `yaml:"options"` + Envs map[string]ErrorGroupsEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type ErrorGroupsEnv struct { + CHID string `yaml:"ch_id"` + Options *ErrorGroupsOptions `yaml:"options"` +} + +type ErrorGroupsOptions struct { + LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` + QueryFilter map[string]string `yaml:"query_filter"` +} + +type AsyncSearch struct { + Options AsyncSearchOptions `yaml:"options"` + Envs map[string]AsyncSearchEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type AsyncSearchEnv struct { + SeqDBID string `yaml:"seq_db_id"` + Options *AsyncSearchOptions `yaml:"options"` +} + +type AsyncSearchOptions struct { + AdminUsers []string `yaml:"admin_users"` + ListQueryLengthLimit int `yaml:"list_query_length_limit"` +} diff --git a/internal/app/config/v2/validate.go b/internal/app/config/v2/validate.go new file mode 100644 index 0000000..023ff2a --- /dev/null +++ b/internal/app/config/v2/validate.go @@ -0,0 +1,449 @@ +package config + +import ( + "fmt" + "time" +) + +const ( + MaxHTTPHeaderBytes = 1 << 12 // 4 KiB + + DefaultSeqDBClientID = "default_seq_db" + DefaultCHClientID = "default_ch" + DefaultInmemID = "default_inmem" + DefaultRedisWithInmemID = "default_redis_with_inmem" + DefaultRedisWithoutInmemID = "default_redis_without_inmem" + + SeqDBClientModeGRPC = "grpc" + + MaskModeMask = "mask" + MaskModeReplace = "replace" + MaskModeCut = "cut" + + FieldFilterConditionAnd = "and" + FieldFilterConditionOr = "or" + FieldFilterConditionNot = "not" + + FieldFilterModeEqual = "equal" + FieldFilterModeContains = "contains" + FieldFilterModePrefix = "prefix" + FieldFilterModeSuffix = "suffix" + + defaultGRPCConnectionTimeout = 120 * time.Second + minGRPCKeepaliveTime = 10 * time.Second + minGRPCKeepaliveTimeout = 1 * time.Second + + defaultAsyncSearchListQueryLengthLimit = 1000 + defaultCORSAllowedOrigins = "*" + + defaultMaxSearchTotal = 1000000 + defaultMaxSearchOffset = 1000000 + defaultMaxExportLimit = 100000 + defaultMaxSearchLimit = 1000 + defaultSeqCLIMaxSearchLimit = 1000 + defaultMaxAggregationsPerRequest = 1 + defaultMaxBucketsPerAggregationTs = 200 + defaultMaxParallelExportRequests = 1 + + defaultInmemCacheNumCounters = 10000000 + defaultInmemCacheMaxCost = 1000000 + defaultInmemCacheBufferItems = 64 + + defaultEventsCacheTTL = 24 * time.Hour + + defaultLogsLifespanCacheTTL = 10 * time.Minute + + defaultAdminCacheTTL = time.Hour + + defaultClickHouseDialTimeout = 5 * time.Second + defaultClickHouseReadTimeout = 30 * time.Second + + defaultUsePreparedStatements = true +) + +var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE"} + +func (cfg *Config) Validate() error { + // seqDBIDs, chIDs, err := validateClients(&cfg.Clients) + // if err != nil { + // return err + // } + // cacheIDs, err := validateCache(&cfg.Cache) + // if err != nil { + // return err + // } + + validateDB(cfg.DB) + + if err := validateServer(&cfg.Server); err != nil { + return err + } + // if err := validateSeqAPI(cfg.Handlers.SeqAPI, seqDBIDs, cacheIDs); err != nil { + // return err + // } + // if err := validateAdmin(cfg.Handlers.Admin, &cfg.Cache); err != nil { + // return err + // } + + // if cfg.Handlers.MassExport != nil { + // me := cfg.Handlers.MassExport + // + // if len(me.Envs) > 0 { + // if me.SeqDBID != "" { + // return fmt.Errorf("handlers.mass_export.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + // } + // if me.DefaultEnv == "" { + // return fmt.Errorf("handlers.mass_export.default_env must be specified when envs is used") + // } + // if _, exists := me.Envs[me.DefaultEnv]; !exists { + // return fmt.Errorf("handlers.mass_export.default_env %q not found in envs", me.DefaultEnv) + // } + // + // for envName, envConfig := range me.Envs { + // if envConfig.SeqDBID == "" { + // return fmt.Errorf("handlers.mass_export.envs[%q].seq_db_id cannot be empty", envName) + // } + // if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + // return fmt.Errorf("unknown handlers.mass_export.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + // } + // + // if envConfig.Options != nil { + // if envConfig.Options.SessionStore.RedisID == "" { + // return fmt.Errorf("handlers.mass_export.envs[%q].options.session_store.redis_id cannot be empty", envName) + // } + // if _, ok := redisIDs[envConfig.Options.SessionStore.RedisID]; !ok { + // return fmt.Errorf("unknown handlers.mass_export.envs[%q].options.session_store.redis_id %q", envName, envConfig.Options.SessionStore.RedisID) + // } + // if envConfig.Options.FileStore.S3 == nil { + // return fmt.Errorf("handlers.mass_export.envs[%q].options.file_store.s3 cannot be empty", envName) + // } + // } + // } + // } else { + // if me.SeqDBID == "" { + // return fmt.Errorf("handlers.mass_export.seq_db_id cannot be empty when envs is not used") + // } + // if _, ok := seqDBIDs[me.SeqDBID]; !ok { + // return fmt.Errorf("unknown handlers.mass_export.seq_db_id %q", me.SeqDBID) + // } + // if me.Options.SessionStore.RedisID == "" { + // return fmt.Errorf("handlers.mass_export.options.session_store.redis_id cannot be empty") + // } + // if _, ok := redisIDs[me.Options.SessionStore.RedisID]; !ok { + // return fmt.Errorf("unknown handlers.mass_export.options.session_store.redis_id %q", me.Options.SessionStore.RedisID) + // } + // } + // } + // + // if cfg.Handlers.AsyncSearch != nil { + // as := cfg.Handlers.AsyncSearch + // + // if as.Options.ListQueryLengthLimit <= 0 { + // as.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + // } + // + // if len(as.Envs) > 0 { + // if as.SeqDBID != "" { + // return fmt.Errorf("handlers.async_search.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") + // } + // if as.DefaultEnv == "" { + // return fmt.Errorf("handlers.async_search.default_env must be specified when envs is used") + // } + // if _, ok := as.Envs[as.DefaultEnv]; !ok { + // return fmt.Errorf("handlers.async_search.default_env %q not found in envs", as.DefaultEnv) + // } + // + // for envName, envConfig := range as.Envs { + // if envConfig.SeqDBID == "" { + // return fmt.Errorf("handlers.async_search.envs[%q].seq_db_id cannot be empty", envName) + // } + // if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + // return fmt.Errorf("unknown handlers.async_search.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) + // } + // if envConfig.Options.ListQueryLengthLimit <= 0 { + // envConfig.Options.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + // } + // } + // } else { + // if as.SeqDBID == "" { + // return fmt.Errorf("handlers.async_search.seq_db_id cannot be empty when envs is not used") + // } + // if _, ok := seqDBIDs[as.SeqDBID]; !ok { + // return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", as.SeqDBID) + // } + // } + // } + // + // if cfg.Handlers.ErrorGroups != nil { + // eg := cfg.Handlers.ErrorGroups + // if len(eg.Envs) > 0 { + // if eg.CHID != "" { + // return fmt.Errorf("handlers.error_groups.ch_id must be empty when envs is used. Put ch_id inside each env") + // } + // if eg.DefaultEnv == "" { + // return fmt.Errorf("handlers.error_groups.default_env must be specified when envs is used") + // } + // if _, ok := eg.Envs[eg.DefaultEnv]; !ok { + // return fmt.Errorf("handlers.error_groups.default_env %q not found in envs", eg.DefaultEnv) + // } + // + // for envName, envConfig := range eg.Envs { + // if envConfig.CHID == "" { + // return fmt.Errorf("handlers.error_groups.envs[%q].ch_id cannot be empty", envName) + // } + // if _, ok := chIDs[envConfig.CHID]; !ok { + // return fmt.Errorf("unknown handlers.error_groups.envs[%q].ch_id %q", envName, envConfig.CHID) + // } + // } + // } else { + // if cfg.Handlers.ErrorGroups.CHID == "" { + // return fmt.Errorf("handlers.error_groups.ch_id cannot be empty when envs is not used") + // } + // if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { + // return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) + // } + // } + // } + + return nil +} + +func validateServer(server *Server) error { + if server.HTTP == nil && server.GRPC == nil { + return fmt.Errorf("server: at least one of http or grpc must be set") + } + + if server.HTTP != nil && server.HTTP.CORS != nil { + if len(server.HTTP.CORS.AllowedOrigins) == 0 { + server.HTTP.CORS.AllowedOrigins = []string{defaultCORSAllowedOrigins} + } + if len(server.HTTP.CORS.AllowedMethods) == 0 { + server.HTTP.CORS.AllowedMethods = defaultCORSAllowedMethods + } + } + + if server.GRPC != nil && server.GRPC.ConnectionTimeout <= 0 { + server.GRPC.ConnectionTimeout = defaultGRPCConnectionTimeout + } + + return nil +} + +// func validateClients(clients *Clients) (seqDBIDs, chIDs map[string]struct{}, err error) { +// if len(clients.SeqDB) == 0 { +// return nil, nil, fmt.Errorf("clients.seq_db must contain at least one client") +// } +// +// seqDBIDs = make(map[string]struct{}, len(clients.SeqDB)) +// for i := range clients.SeqDB { +// c := &clients.SeqDB[i] +// if c.ID == "" { +// return nil, nil, fmt.Errorf("clients.seq_db[%d].id cannot be empty", i) +// } +// if _, ok := seqDBIDs[c.ID]; ok { +// return nil, nil, fmt.Errorf("duplicate clients.seq_db.id %q", c.ID) +// } +// +// seqDBIDs[c.ID] = struct{}{} +// +// if len(c.Addrs) == 0 { +// return nil, nil, fmt.Errorf("clients.seq_db[%q].addrs cannot be empty", c.ID) +// } +// if c.Timeout <= 0 { +// return nil, nil, fmt.Errorf("clients.seq_db[%q].timeout must be > 0", c.ID) +// } +// if c.ClientMode == "" { +// c.ClientMode = SeqDBClientModeGRPC +// } else if c.ClientMode != SeqDBClientModeGRPC { +// return nil, nil, fmt.Errorf("invalid clients.seq_db[%q].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, SeqDBClientModeGRPC) +// } +// +// if c.GRPCParams != nil { +// if c.GRPCParams.Keepalive != nil { +// c.GRPCParams.Keepalive.Time = max(c.GRPCParams.Keepalive.Time, minGRPCKeepaliveTime) +// c.GRPCParams.Keepalive.Timeout = max(c.GRPCParams.Keepalive.Timeout, minGRPCKeepaliveTimeout) +// } +// } +// } +// +// chIDs = make(map[string]struct{}, len(clients.ClickHouse)) +// for i := range clients.ClickHouse { +// ch := &clients.ClickHouse[i] +// if ch.ID == "" { +// return nil, nil, fmt.Errorf("clients.clickhouse[%d].id cannot be empty", i) +// } +// if _, ok := chIDs[ch.ID]; ok { +// return nil, nil, fmt.Errorf("duplicate clients.clickhouse.id %q", ch.ID) +// } +// +// chIDs[ch.ID] = struct{}{} +// +// if ch.DialTimeout <= 0 { +// ch.DialTimeout = defaultClickHouseDialTimeout +// } +// if ch.ReadTimeout <= 0 { +// ch.ReadTimeout = defaultClickHouseReadTimeout +// } +// } +// +// return seqDBIDs, chIDs, nil +// } +// +// func validateCache(cache *Cache) (map[string]struct{}, error) { +// inmemIDs := make(map[string]struct{}, len(cache.Inmemory)) +// for i := range cache.Inmemory { +// inm := &cache.Inmemory[i] +// if inm.ID == "" { +// return nil, fmt.Errorf("cache.inmemory[%d].id cannot be empty", i) +// } +// if _, ok := inmemIDs[inm.ID]; ok { +// return nil, fmt.Errorf("duplicate cache.inmemory.id %q", inm.ID) +// } +// +// inmemIDs[inm.ID] = struct{}{} +// +// if inm.NumCounters <= 0 { +// inm.NumCounters = defaultInmemCacheNumCounters +// } +// if inm.MaxCost <= 0 { +// inm.MaxCost = defaultInmemCacheMaxCost +// } +// if inm.BufferItems <= 0 { +// inm.BufferItems = defaultInmemCacheBufferItems +// } +// } +// +// cacheIDs := make(map[string]struct{}, len(cache.Inmemory)+len(cache.Redis)) +// for i := range cache.Redis { +// r := &cache.Redis[i] +// if r.ID == "" { +// return nil, fmt.Errorf("cache.redis[%d].id cannot be empty", i) +// } +// if _, ok := cacheIDs[r.ID]; ok { +// return nil, fmt.Errorf("duplicate cache.redis.id %q", r.ID) +// } +// +// cacheIDs[r.ID] = struct{}{} +// +// if r.WithInmemID != "" { +// if _, ok := inmemIDs[r.WithInmemID]; !ok { +// return nil, fmt.Errorf("cache.redis[%q].with_inmem_id %q not found in cache.inmemory", r.ID, r.WithInmemID) +// } +// } +// +// for id := range inmemIDs { +// if !slices.ContainsFunc(cache.Redis, func(r Redis) bool { +// return r.WithInmemID == id +// }) { +// cacheIDs[id] = struct{}{} +// } +// } +// } +// +// return cacheIDs, nil +// } + +func validateDB(db *DB) { + if db != nil && db.UsePreparedStatements == nil { + v := defaultUsePreparedStatements + db.UsePreparedStatements = &v + } +} + +// func validateSeqAPI(seqAPI *SeqAPI, seqDBIDs, cacheIDs map[string]struct{}) error { +// setSeqAPIOptionsDefaults(&seqAPI.Options) +// +// _ = cacheIDs // Проверки потом нафигачу на кэши, пусть весит пока голенький +// if len(seqAPI.Envs) > 0 { +// if seqAPI.SeqDBID != "" { +// return fmt.Errorf("handlers.seq_api.seq_db_id must be empty when envs is used. Put seq_db_id inside each env") +// } +// if seqAPI.DefaultEnv == "" { +// return fmt.Errorf("handlers.seq_api.default_env must be specified when envs is used") +// } +// if _, exists := seqAPI.Envs[seqAPI.DefaultEnv]; !exists { +// return fmt.Errorf("handlers.seq_api.default_env %q not found in envs", seqAPI.DefaultEnv) +// } +// +// for envName, envConfig := range seqAPI.Envs { +// if envConfig.SeqDBID == "" { +// return fmt.Errorf("handlers.seq_api.envs[%q].seq_db_id cannot be empty", envName) +// } +// if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { +// return fmt.Errorf("unknown handlers.seq_api.envs[%q].seq_db_id %q", envName, envConfig.SeqDBID) +// } +// setSeqAPIOptionsDefaults(envConfig.Options) +// } +// } else { +// if seqAPI.SeqDBID == "" { +// return fmt.Errorf("handlers.seq_api.seq_db_id cannot be empty when envs is not used") +// } +// if _, ok := seqDBIDs[seqAPI.SeqDBID]; !ok { +// return fmt.Errorf("unknown handlers.seq_api.seq_db_id %q", seqAPI.SeqDBID) +// } +// } +// +// return nil +// } + +// func validateAdmin(admin *Admin, cache *Cache) error { +// if admin == nil { +// return nil +// } +// +// if admin.Options.Cache != nil { +// if admin.Options.Cache.ID == "" { +// return fmt.Errorf("handlers.admin.options.cache.id cannot be empty") +// } +// redisCfg := cache.RedisByID(admin.Options.Cache.ID) +// if redisCfg == nil { +// return fmt.Errorf("unknown handlers.admin.options.cache.id %q", admin.Options.Cache.ID) +// } +// if redisCfg == nil { +// return fmt.Errorf("unknown handlers.admin.options.cache.id %q", admin.Options.Cache.ID) +// } +// if redisCfg.WithInmemID != "" { +// return fmt.Errorf("handlers.admin.options.cache.id %q: with_inmem_id is not allowed", admin.Options.Cache.ID) +// } +// if admin.Options.Cache.TTL <= 0 { +// admin.Options.Cache.TTL = defaultAdminCacheTTL +// } +// } +// +// if len(admin.Envs) > 0 { +// +// } +// } +// +// func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { +// if options.Limits.MaxAggregationsPerRequest <= 0 { +// options.Limits.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest +// } +// if options.Limits.MaxBucketsPerAggregationTs <= 0 { +// options.Limits.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs +// } +// if options.Limits.MaxParallelExportRequests <= 0 { +// options.Limits.MaxParallelExportRequests = defaultMaxParallelExportRequests +// } +// if options.Limits.MaxSearchTotal <= 0 { +// options.Limits.MaxSearchTotal = defaultMaxSearchTotal +// } +// if options.Limits.MaxSearchOffset <= 0 { +// options.Limits.MaxSearchOffset = defaultMaxSearchOffset +// } +// if options.Limits.MaxExportLimit <= 0 { +// options.Limits.MaxExportLimit = defaultMaxExportLimit +// } +// if options.Limits.MaxSearchLimit <= 0 { +// options.Limits.MaxSearchLimit = defaultMaxSearchLimit +// } +// if options.Limits.SeqCLIMaxSearchLimit <= 0 { +// options.Limits.SeqCLIMaxSearchLimit = defaultSeqCLIMaxSearchLimit +// } +// if options.Caches.Events.TTL <= 0 { +// options.Caches.Events.TTL = defaultEventsCacheTTL +// } +// if options.Caches.LogsLifespan.TTL <= 0 { +// options.Caches.LogsLifespan.TTL = defaultLogsLifespanCacheTTL +// } +// } diff --git a/internal/app/mw/auth.go b/internal/app/mw/auth.go index a927e2d..bb16046 100644 --- a/internal/app/mw/auth.go +++ b/internal/app/mw/auth.go @@ -7,7 +7,8 @@ import ( "strings" "github.com/ozontech/seq-ui/internal/app/auth" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/logger" ) @@ -28,26 +29,32 @@ type AuthProviders struct { OidcProvider auth.OIDCProvider } -func NewAuthProviders( - ctx context.Context, - jwtSecretKey string, oidcCfg *config.OIDC, - cacheCfg config.Cache, -) (AuthProviders, error) { - authPrvds := AuthProviders{} +func (p *AuthProviders) enabled() bool { + return p.JwtProvider != nil || p.OidcProvider != nil +} + +// AuthProvidersByEnv contains auth providers by env, empty env is root options. +type AuthProvidersByEnv map[string]AuthProviders - if jwtSecretKey == "" && oidcCfg == nil { - return authPrvds, nil +func (providers AuthProvidersByEnv) get(ctx context.Context) AuthProviders { + if provider, ok := providers[types.GetEnvKey(ctx)]; ok { + return provider } + return providers[""] +} + +func NewAuthProvider(ctx context.Context, options config.AuthOptions) (AuthProviders, error) { + authPrvds := AuthProviders{} - if jwtSecretKey != "" { + if options.JWT != nil { logger.Info("initializing jwt provider") - authPrvds.JwtProvider = auth.NewJWTProvider(jwtSecretKey) + authPrvds.JwtProvider = auth.NewJWTProvider(options.JWT) } - if oidcCfg != nil { + if options.OIDC != nil { logger.Info("initializing oidc provider") var err error - authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, oidcCfg, cacheCfg) + authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, options.OIDC) if err != nil { return authPrvds, fmt.Errorf("failed to init oidc provider: %w", err) } diff --git a/internal/app/mw/env.go b/internal/app/mw/env.go new file mode 100644 index 0000000..1f15ba8 --- /dev/null +++ b/internal/app/mw/env.go @@ -0,0 +1,69 @@ +package mw + +import ( + "fmt" + + "github.com/ozontech/seq-ui/internal/app/config/v2" +) + +// HandlerEnvs describes envs of the handler. +type handlerEnvs struct { + DefaultEnv string + Envs map[string]struct{} +} + +// EnvResolver resolves env of the request for auth and rate limiting by api name. +// APIs not present in the resolver (userprofile, dashboards) or handlers without envs +// always use root options (empty env), env from the request is ignored for them. +type EnvResolver map[string]handlerEnvs + +// newEnvResolver collects envs of handlers that support envs. +func NewEnvResolver(handlers *config.Handlers) EnvResolver { + r := EnvResolver{} + + if h := handlers.SeqAPI; h != nil { + envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + r["seqapi"], r["SeqAPIService"] = envs, envs + } + if h := handlers.ErrorGroups; h != nil { + envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + r["errorgroups"], r["ErrorGroupsService"] = envs, envs + } + if h := handlers.MassExport; h != nil { + envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + r["massexport"], r["MassExportService"] = envs, envs + } + if h := handlers.Admin; h != nil { + envs := handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + r["admin"], r["AdminService"] = envs, envs + } + if h := handlers.AsyncSearch; h != nil { + r["async_search"] = handlerEnvs{DefaultEnv: h.DefaultEnv, Envs: envNames(h.Envs)} + } + return r +} + +func (resolver EnvResolver) resolve(api, env string) (string, error) { + handler, ok := resolver[api] + if !ok || len(handler.Envs) == 0 { + return "", nil + } + + if env == "" { + return handler.DefaultEnv, nil + } + + if _, ok := handler.Envs[env]; !ok { + return "", fmt.Errorf("env %q not found", env) + } + + return env, nil +} + +func envNames[T any](envs map[string]T) map[string]struct{} { + names := make(map[string]struct{}, len(envs)) + for name := range envs { + names[name] = struct{}{} + } + return names +} diff --git a/internal/app/mw/grpc_mw.go b/internal/app/mw/grpc_mw.go index a76e6f2..5d0e52d 100644 --- a/internal/app/mw/grpc_mw.go +++ b/internal/app/mw/grpc_mw.go @@ -3,6 +3,7 @@ package mw import ( "context" "fmt" + "strings" "time" "go.opentelemetry.io/otel/attribute" @@ -138,7 +139,7 @@ var noAuthGRPCMethods = map[string]map[string]struct{}{ var errUnauth = status.Error(codes.Unauthenticated, types.ErrUnauthenticated.Error()) -func GRPCAuthInterceptor(providers *AuthProviders) grpc.UnaryServerInterceptor { +func GRPCAuthInterceptor(providers AuthProvidersByEnv) grpc.UnaryServerInterceptor { return func( ctx context.Context, req any, info *grpc.UnaryServerInfo, h grpc.UnaryHandler, @@ -151,6 +152,10 @@ func GRPCAuthInterceptor(providers *AuthProviders) grpc.UnaryServerInterceptor { if _, noAuth := noAuthGRPCMethods[svc][method]; noAuth { return h(ctx, req) } + provider := providers.get(ctx) + if !provider.enabled() { + return h(ctx, req) + } md, ok := metadata.FromIncomingContext(ctx) if !ok { logger.Error("no metadata provided") @@ -161,7 +166,7 @@ func GRPCAuthInterceptor(providers *AuthProviders) grpc.UnaryServerInterceptor { logger.Error("no authorization metadata provided") return nil, errUnauth } - userName, err := providers.auth(ctx, authHeaderSlice[0]) + userName, err := provider.auth(ctx, authHeaderSlice[0]) if err != nil { logger.Error("token auth failed", zap.Error(err)) return nil, errUnauth @@ -189,7 +194,7 @@ func GRPCRecoverInterceptor() grpc.UnaryServerInterceptor { } } -func GRPCRateLimitInterceptor(rateLimiters map[string]map[string]RateLimiter) grpc.UnaryServerInterceptor { +func GRPCRateLimitInterceptor(rateLimiters RateLimitersByEnv) grpc.UnaryServerInterceptor { return func( ctx context.Context, req any, info *grpc.UnaryServerInfo, h grpc.UnaryHandler, @@ -206,7 +211,7 @@ func GRPCRateLimitInterceptor(rateLimiters map[string]map[string]RateLimiter) gr return h(ctx, req) } - userToRateLimiter, ok := rateLimiters[svc] + userToRateLimiter, ok := rateLimiters.get(ctx)[svc] if !ok { return h(ctx, req) } @@ -244,3 +249,37 @@ func GRPCProcessHeadersInterceptor() grpc.UnaryServerInterceptor { return h(ctx, req) } } + +func GRPCEnvInterceptor(resolver EnvResolver) grpc.UnaryServerInterceptor { + return func( + ctx context.Context, req any, info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, + ) (resp any, err error) { + svc, method, err := parseGRPCFullMethod(info.FullMethod) + if err != nil { + msg := "failed to parse gRPC FullMethod" + logger.Error(msg, zap.Error(err)) + return nil, status.Error(codes.Internal, msg) + } + + var reqEnv string + if md, ok := metadata.FromIncomingContext(ctx); ok { + if values := md.Get("env"); len(values) > 0 { + reqEnv = values[0] + } + } + + api := svc + // async search methods are part of SeqAPIService, but have own envs + if svc == "SeqAPIService" && strings.Contains(method, "AsyncSearch") { + api = "async_search" + } + + env, err := resolver.resolve(api, reqEnv) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + return handler(types.SetEnvKey(ctx, env), req) + } +} diff --git a/internal/app/mw/http_mw.go b/internal/app/mw/http_mw.go index 7d4e1e6..4d65569 100644 --- a/internal/app/mw/http_mw.go +++ b/internal/app/mw/http_mw.go @@ -142,7 +142,7 @@ func HTTPTraceInterceptor() func(next http.Handler) http.Handler { } } -func HTTPAuthInterceptor(providers *AuthProviders) func(next http.Handler) http.Handler { +func HTTPAuthInterceptor(providers AuthProvidersByEnv) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -157,8 +157,14 @@ func HTTPAuthInterceptor(providers *AuthProviders) func(next http.Handler) http. return } + provider := providers.get(ctx) + if !provider.enabled() { + next.ServeHTTP(w, r) + return + } + authHeader := r.Header.Get("Authorization") - username, err := providers.auth(ctx, authHeader) + username, err := provider.auth(ctx, authHeader) if err != nil { logger.Error("token auth failed", zap.Error(err)) http.Error(w, "Unauthenticated", http.StatusUnauthorized) @@ -191,7 +197,7 @@ func HTTPRecoverInterceptor() func(next http.Handler) http.Handler { } } -func HTTPRateLimitInterceptor(rateLimiters map[string]map[string]RateLimiter) func(next http.Handler) http.Handler { +func HTTPRateLimitInterceptor(rateLimiters RateLimitersByEnv) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -213,7 +219,7 @@ func HTTPRateLimitInterceptor(rateLimiters map[string]map[string]RateLimiter) fu http.NotFound(w, r) return } - userToRateLimiter, ok := rateLimiters[uriApi] + userToRateLimiter, ok := rateLimiters.get(ctx)[uriApi] if !ok { next.ServeHTTP(w, r) return @@ -271,3 +277,30 @@ func HTTPProcessHeadersInterceptor() func(next http.Handler) http.Handler { return http.HandlerFunc(fn) } } + +func HTTPEnvInterceptor(resolver EnvResolver) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + uriApi, _, uriMethod, err := parseURI(r.RequestURI) + if err != nil { + logger.Error("failed to parse URI", zap.Error(err)) + http.NotFound(w, r) + return + } + + // async search routes are part of seqapi, but have own envs + if uriApi == "seqapi" && uriMethod == "async_search" { + uriApi = "async_search" + } + + env, err := resolver.resolve(uriApi, r.URL.Query().Get("env")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + next.ServeHTTP(w, r.WithContext(types.SetEnvKey(r.Context(), env))) + } + return http.HandlerFunc(fn) + } +} diff --git a/internal/app/mw/ratelimiter.go b/internal/app/mw/ratelimiter.go index f181616..e2b67b5 100644 --- a/internal/app/mw/ratelimiter.go +++ b/internal/app/mw/ratelimiter.go @@ -7,7 +7,7 @@ import ( "net/http" "strconv" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/ratelimiter" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/metric" @@ -22,6 +22,7 @@ var grpcServices = map[string]struct{}{ "DashboardsService": {}, "MassExportService": {}, "ErrorGroupsService": {}, + "AdminService": {}, } type rateLimiter interface { @@ -33,7 +34,41 @@ type RateLimiter struct { perHandler bool } -func NewRateLimiter(api string, cfg config.RateLimiter) (RateLimiter, error) { +// RateLimitersByEnv contains rate limiters by env->api->user, empty env is root options. +type RateLimitersByEnv map[string]map[string]map[string]RateLimiter + +func (ratelimiters RateLimitersByEnv) get(ctx context.Context) map[string]map[string]RateLimiter { + if rl, ok := ratelimiters[types.GetEnvKey(ctx)]; ok { + return rl + } + return ratelimiters[""] +} + +func NewRateLimiters(apiToRateLimiters config.ApiToRateLimiters) (map[string]map[string]RateLimiter, error) { + result := make(map[string]map[string]RateLimiter, len(apiToRateLimiters)) + + for apiName, rateLimiters := range apiToRateLimiters { + result[apiName] = make(map[string]RateLimiter) + + defaultLimiter, err := newRateLimiter(apiName, rateLimiters.Default) + if err != nil { + return nil, fmt.Errorf("init default rate limiter: %w", err) + } + result[apiName][RateLimiterDefaultUser] = defaultLimiter + + for user, userRateLimiter := range rateLimiters.SpecialUsers { + limiter, err := newRateLimiter(apiName, userRateLimiter) + if err != nil { + return nil, fmt.Errorf("init user %q rate limiter: %w", user, err) + } + result[apiName][user] = limiter + } + } + + return result, nil +} + +func newRateLimiter(api string, cfg config.RateLimiter) (RateLimiter, error) { if _, ok := grpcServices[api]; !ok { spec := swagger.GetSpecEx() if !spec.HasApi(api) { diff --git a/internal/app/ratelimiter/ratelimiter.go b/internal/app/ratelimiter/ratelimiter.go index dadee28..47ed459 100644 --- a/internal/app/ratelimiter/ratelimiter.go +++ b/internal/app/ratelimiter/ratelimiter.go @@ -8,7 +8,7 @@ import ( "github.com/throttled/throttled/v2" "github.com/throttled/throttled/v2/store/memstore" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) const rateLimiterQuantityPerCall = 1 diff --git a/internal/app/server/init.go b/internal/app/server/init.go index 26d31a7..b21a3e4 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -5,106 +5,100 @@ import ( "fmt" "net" "net/http" + "time" "github.com/go-chi/chi/v5" "github.com/go-chi/cors" grpc_mw "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/prometheus/client_golang/prometheus/promhttp" - "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/reflection" "github.com/ozontech/seq-ui/internal/api" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/mw" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/tracing" ) -const ( - defaultCORSAllowedOrigins = "*" - maxHTTPHeaderBytes = 1 << 12 // 4 KiB -) - -var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE"} - func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { - var err error - - s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.JWTSecretKey, s.config.OIDC, s.config.Cache) - if err != nil { + if err := s.prepareAuth(ctx); err != nil { return err } - - err = s.prepareRateLimiters() - if err != nil { + if err := s.prepareRateLimiters(); err != nil { return err } s.prepareGRPCServer(registrar) - s.prepareHTTPServer(ctx, registrar) - err = s.prepareDebugServer(ctx) - if err != nil { + if err := s.prepareDebugServer(ctx); err != nil { + return err + } + + return nil +} + +func (s *Server) prepareAuth(ctx context.Context) error { + if s.config.Auth == nil { return nil } + s.authPrvds = make(mw.AuthProvidersByEnv, len(s.config.Auth.Envs)+1) + + provider, err := mw.NewAuthProvider(ctx, s.config.Auth.Options) + if err != nil { + return fmt.Errorf("init auth: %w", err) + } + s.authPrvds[""] = provider + + for envName, envConfig := range s.config.Auth.Envs { + provider, err := mw.NewAuthProvider(ctx, envConfig) + if err != nil { + return fmt.Errorf("init auth for env %q: %w", envName, err) + } + s.authPrvds[envName] = provider + } return nil } // setupCORS applies CORS policies set in config to the provided mux. -func (s *Server) setupCORS(mux *chi.Mux) { - allowedOrigins := s.config.CORS.AllowedOrigins - if len(allowedOrigins) == 0 { - allowedOrigins = append(allowedOrigins, defaultCORSAllowedOrigins) - } - allowedMethods := s.config.CORS.AllowedMethods - if len(allowedMethods) == 0 { - allowedMethods = append(allowedMethods, defaultCORSAllowedMethods...) +func (s *Server) setupCORS(mux *chi.Mux, cfg *config.CORS) { + if cfg == nil { + return } + mux.Use(cors.Handler(cors.Options{ - AllowedOrigins: allowedOrigins, - AllowedMethods: allowedMethods, - AllowedHeaders: s.config.CORS.AllowedHeaders, - ExposedHeaders: s.config.CORS.ExposedHeaders, - AllowCredentials: s.config.CORS.AllowCredentials, - MaxAge: s.config.CORS.MaxAge, - OptionsPassthrough: s.config.CORS.OptionsPassthrough, + AllowedOrigins: cfg.AllowedOrigins, + AllowedMethods: cfg.AllowedMethods, + AllowedHeaders: cfg.AllowedHeaders, + ExposedHeaders: cfg.ExposedHeaders, + AllowCredentials: cfg.AllowCredentials, + MaxAge: cfg.MaxAge, + OptionsPassthrough: cfg.OptionsPassthrough, })) } // prepareRateLimiters prepares requests rate limiters based on server config. func (s *Server) prepareRateLimiters() error { - if len(s.config.RateLimiters) == 0 { + if s.config.RateLimiting == nil { return nil } - s.rateLimiters = make(map[string]map[string]mw.RateLimiter) + s.rateLimiters = make(mw.RateLimitersByEnv, len(s.config.RateLimiting.Envs)+1) - for apiName, rateLimiters := range s.config.RateLimiters { - s.rateLimiters[apiName] = make(map[string]mw.RateLimiter) + limiters, err := mw.NewRateLimiters(s.config.RateLimiting.Options) + if err != nil { + return fmt.Errorf("init rate limiter: %w", err) + } + s.rateLimiters[""] = limiters - logger.Info("init default rate limiter", zap.String("api", apiName)) - defaultLimiter, err := mw.NewRateLimiter(apiName, rateLimiters.Default) + for envName, envConfig := range s.config.RateLimiting.Envs { + limiters, err := mw.NewRateLimiters(envConfig) if err != nil { - return fmt.Errorf("init default rate limiter: %w", err) - } - - s.rateLimiters[apiName][mw.RateLimiterDefaultUser] = defaultLimiter - - for user, rateLimiter := range rateLimiters.SpecialUsers { - logger.Info( - "init user rate limiter", - zap.String("api", apiName), - zap.String("user", user), - ) - limiter, err := mw.NewRateLimiter(apiName, rateLimiter) - if err != nil { - return fmt.Errorf("init user %q rate limiter: %w", user, err) - } - - s.rateLimiters[apiName][user] = limiter + return fmt.Errorf("init rate limiter for env %q: %w", envName, err) } + s.rateLimiters[envName] = limiters } return nil @@ -112,12 +106,17 @@ func (s *Server) prepareRateLimiters() error { // prepareGRPCServer creates gRPC server and binds it to the Server instance. func (s *Server) prepareGRPCServer(registrar *api.Registrar) { + if s.config.GRPC == nil { + return + } + interceptors := []grpc.UnaryServerInterceptor{ mw.GRPCRecoverInterceptor(), mw.GRPCProcessHeadersInterceptor(), + mw.GRPCEnvInterceptor(s.envResolver), } - if s.authPrvds.JwtProvider != nil || s.authPrvds.OidcProvider != nil { - interceptors = append(interceptors, mw.GRPCAuthInterceptor(&s.authPrvds)) + if len(s.authPrvds) > 0 { + interceptors = append(interceptors, mw.GRPCAuthInterceptor(s.authPrvds)) } interceptors = append(interceptors, mw.GRPCMetricInterceptor(), @@ -130,7 +129,7 @@ func (s *Server) prepareGRPCServer(registrar *api.Registrar) { opts := []grpc.ServerOption{ grpc.UnaryInterceptor(grpc_mw.ChainUnaryServer(interceptors...)), - grpc.ConnectionTimeout(s.config.GRPCConnectionTimeout), + grpc.ConnectionTimeout(s.config.GRPC.ConnectionTimeout), } s.grpcServer = grpc.NewServer(opts...) @@ -142,17 +141,21 @@ func (s *Server) prepareGRPCServer(registrar *api.Registrar) { // prepareHTTPServer prepares HTTP server with applied CORS policies and added interceptors. func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar) { - mux := chi.NewMux() - if s.config.CORS != nil { - s.setupCORS(mux) + if s.config.HTTP == nil { + return } + + mux := chi.NewMux() + s.setupCORS(mux, s.config.HTTP.CORS) + interceptors := chi.Middlewares{ mw.HTTPNotFoundInterceptor(), mw.HTTPRecoverInterceptor(), mw.HTTPProcessHeadersInterceptor(), + mw.HTTPEnvInterceptor(s.envResolver), } - if s.authPrvds.JwtProvider != nil || s.authPrvds.OidcProvider != nil { - interceptors = append(interceptors, mw.HTTPAuthInterceptor(&s.authPrvds)) + if len(s.authPrvds) > 0 { + interceptors = append(interceptors, mw.HTTPAuthInterceptor(s.authPrvds)) } interceptors = append(interceptors, mw.HTTPMetricInterceptor(), @@ -166,39 +169,40 @@ func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar registrar.RegisterHTTPHandlers(mux) - s.httpServer = s.makeHTTPServer(ctx, mux) + s.httpServer = s.makeHTTPServer(ctx, mux, s.config.HTTP.ReadHeaderTimeout, s.config.HTTP.ReadTimeout, s.config.HTTP.WriteTimeout) } // prepareHTTPMux prepares debug HTTP server mux with applied CORS policies and added interceptors. func (s *Server) prepareDebugServer(ctx context.Context) error { mux := chi.NewMux() - if s.config.CORS != nil { - s.setupCORS(mux) - } + s.setupCORS(mux, s.config.Debug.CORS) interceptors := chi.Middlewares{ mw.HTTPRecoverInterceptor(), } mux.Use(interceptors...) mux.Handle("/metrics", promhttp.Handler()) serveHealth(mux) - _, port, err := net.SplitHostPort(s.config.HTTPAddr) - if err != nil { - return err + // swagger describes http api, so it is served only if http server is configured + if s.config.HTTP != nil { + _, port, err := net.SplitHostPort(s.config.HTTP.Addr) + if err != nil { + return err + } + serveSwaggerUI(mux, port) } - serveSwaggerUI(mux, port) servePprof(mux) - s.debugServer = s.makeHTTPServer(ctx, mux) + s.debugServer = s.makeHTTPServer(ctx, mux, s.config.Debug.ReadHeaderTimeout, s.config.Debug.ReadTimeout, s.config.Debug.WriteTimeout) return nil } // makeHTTPServer makes HTTP server from provided mux. -func (s *Server) makeHTTPServer(ctx context.Context, mux *chi.Mux) *http.Server { +func (s *Server) makeHTTPServer(ctx context.Context, mux *chi.Mux, readHeaderTimeout, readTimeout, writeTimeout time.Duration) *http.Server { return &http.Server{ Handler: mux, - ReadHeaderTimeout: s.config.HTTPReadHeaderTimeout, - ReadTimeout: s.config.HTTPReadTimeout, - WriteTimeout: s.config.HTTPWriteTimeout, - MaxHeaderBytes: maxHTTPHeaderBytes, // 4 KiB + ReadHeaderTimeout: readHeaderTimeout, + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + MaxHeaderBytes: config.MaxHTTPHeaderBytes, // 4 KiB BaseContext: func(_ net.Listener) context.Context { return ctx }, diff --git a/internal/app/server/run.go b/internal/app/server/run.go index ce7dc2c..78c9e78 100644 --- a/internal/app/server/run.go +++ b/internal/app/server/run.go @@ -17,28 +17,32 @@ func (s *Server) Run(ctx context.Context) error { errWg, ctx := errgroup.WithContext(ctx) // run gRPC server - errWg.Go(func() error { - lis, err := net.Listen("tcp", s.config.GRPCAddr) - if err != nil { - return err - } - - return s.grpcServer.Serve(lis) - }) + if s.grpcServer != nil { + errWg.Go(func() error { + lis, err := net.Listen("tcp", s.config.GRPC.Addr) + if err != nil { + return err + } + + return s.grpcServer.Serve(lis) + }) + } // run HTTP server - errWg.Go(func() error { - l, err := net.Listen("tcp", s.config.HTTPAddr) - if err != nil { - return err - } - - return s.httpServer.Serve(l) - }) + if s.httpServer != nil { + errWg.Go(func() error { + l, err := net.Listen("tcp", s.config.HTTP.Addr) + if err != nil { + return err + } + + return s.httpServer.Serve(l) + }) + } // run debug server errWg.Go(func() error { - l, err := net.Listen("tcp", s.config.DebugAddr) + l, err := net.Listen("tcp", s.config.Debug.Addr) if err != nil { return err } @@ -47,13 +51,18 @@ func (s *Server) Run(ctx context.Context) error { }) errWg.Go(func() error { - swaggerAddr := s.config.DebugAddr + swaggerUIPrefix - logger.Info("app started", - zap.String("http", s.config.HTTPAddr), - zap.String("debug", s.config.DebugAddr), - zap.String("swagger", swaggerAddr), - zap.String("grpc", s.config.GRPCAddr), - ) + fields := []zap.Field{zap.String("debug", s.config.Debug.Addr)} + if s.httpServer != nil { + fields = append(fields, + zap.String("http", s.config.HTTP.Addr), + zap.String("swagger", s.config.Debug.Addr+swaggerUIPrefix), + ) + } + if s.grpcServer != nil { + fields = append(fields, zap.String("grpc", s.config.GRPC.Addr)) + } + + logger.Info("app started", fields...) return nil }) @@ -71,43 +80,42 @@ func (s *Server) Run(ctx context.Context) error { // Stop the gRPC and HTTP servers. func (s *Server) stop() { - var wg sync.WaitGroup - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - wg.Add(1) - go func() { - if err := s.httpServer.Shutdown(ctx); err != nil { - logger.Error("shutting down the http server", zap.Error(err)) - } else { - logger.Warn("http server gracefully stopped") - } - wg.Done() - }() + var wg sync.WaitGroup + if s.httpServer != nil { + wg.Go(func() { + if err := s.httpServer.Shutdown(ctx); err != nil { + logger.Error("shutting down the http server", zap.Error(err)) + } else { + logger.Warn("http server gracefully stopped") + } + }) + } - wg.Add(1) - go func() { + wg.Go(func() { if err := s.debugServer.Shutdown(ctx); err != nil { logger.Error("shutting down the debug server", zap.Error(err)) } else { logger.Warn("debug server gracefully stopped") } - wg.Done() - }() - - done := make(chan struct{}) - go func() { - s.grpcServer.GracefulStop() - close(done) - }() - select { - case <-done: - logger.Warn("grpc server gracefully stopped") - case <-ctx.Done(): - logger.Error("shutting down the grpc server", zap.Error(ctx.Err())) - logger.Warn("grpc server force stop") - s.grpcServer.Stop() + }) + + if s.grpcServer != nil { + done := make(chan struct{}) + go func() { + s.grpcServer.GracefulStop() + close(done) + }() + select { + case <-done: + logger.Warn("grpc server gracefully stopped") + case <-ctx.Done(): + logger.Error("shutting down the grpc server", zap.Error(ctx.Err())) + logger.Warn("grpc server force stop") + s.grpcServer.Stop() + } } wg.Wait() } diff --git a/internal/app/server/server.go b/internal/app/server/server.go index 6872959..19ab581 100644 --- a/internal/app/server/server.go +++ b/internal/app/server/server.go @@ -8,7 +8,7 @@ import ( "google.golang.org/grpc" "github.com/ozontech/seq-ui/internal/api" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/mw" ) @@ -19,13 +19,17 @@ type Server struct { grpcServer *grpc.Server httpServer *http.Server - authPrvds mw.AuthProviders - rateLimiters map[string]map[string]mw.RateLimiter // rate limiter by api and user + envResolver mw.EnvResolver + authPrvds mw.AuthProvidersByEnv + rateLimiters mw.RateLimitersByEnv } // New returns a new Server. -func New(ctx context.Context, cfg *config.Server, registrar *api.Registrar) (*Server, error) { - s := &Server{config: cfg} +func New(ctx context.Context, cfg *config.Config, registrar *api.Registrar) (*Server, error) { + s := &Server{ + config: &cfg.Server, + envResolver: mw.NewEnvResolver(&cfg.Handlers), + } if err := s.init(ctx, registrar); err != nil { return nil, fmt.Errorf("init server: %w", err) diff --git a/internal/app/server/tech.go b/internal/app/server/tech.go index 4e612d9..954684b 100644 --- a/internal/app/server/tech.go +++ b/internal/app/server/tech.go @@ -15,6 +15,9 @@ import ( "github.com/ozontech/seq-ui/swagger" ) +const swaggerUIPrefix = "/docs/" +const swaggerJSONPath = "/swagger.json" + func serveHealth(mux *chi.Mux) { mux.HandleFunc("/live", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("{}")) @@ -24,9 +27,6 @@ func serveHealth(mux *chi.Mux) { }) } -const swaggerUIPrefix = "/docs/" -const swaggerJSONPath = "/swagger.json" - func defSwagger(host string) ([]byte, error) { s := swagger.GetSpec() s.Host = host @@ -55,6 +55,8 @@ func serveSwaggerUI(mux *chi.Mux, apiPort string) { def, err := defSwagger(r.Host) if err != nil { logger.Error("failed to get swagger def", zap.Error(err)) + http.Error(w, "failed to get swagger def", http.StatusInternalServerError) + return } w.Header().Set("Content-Type", "application/json") diff --git a/internal/app/types/admin.go b/internal/app/types/admin.go index bf4b9b5..0cd8b32 100644 --- a/internal/app/types/admin.go +++ b/internal/app/types/admin.go @@ -27,6 +27,7 @@ type CreateRoleResponse struct { type AddUsersToRoleRequest struct { RoleID int32 Usernames []string + Env string } type GetRolesResponse struct { @@ -35,6 +36,7 @@ type GetRolesResponse struct { type GetRoleRequest struct { RoleID int32 + Env string } type GetRoleResponse struct { @@ -50,10 +52,12 @@ type UpdateRoleRequest struct { type DeleteRoleRequest struct { RoleID int32 ReplacementRoleID *int32 + Env string } type GetUserPermissionsRequest struct { Username string + Env string } type GetUserPermissionsResponse struct { @@ -63,4 +67,5 @@ type GetUserPermissionsResponse struct { type DeleteUsersFromRoleRequest struct { RoleID int32 Usernames []string + Env string } diff --git a/internal/app/types/async_searches.go b/internal/app/types/async_searches.go index 9618a71..75ebb6e 100644 --- a/internal/app/types/async_searches.go +++ b/internal/app/types/async_searches.go @@ -7,6 +7,7 @@ type SaveAsyncSearchRequest struct { OwnerID int64 ExpiresAt time.Time Meta string + Env string } type AsyncSearchInfo struct { @@ -22,4 +23,19 @@ type GetAsyncSearchesListRequest struct { Limit int32 Offset int32 Owner *string + Env string +} + +type GetAsyncSearchByIdRequest struct { + SearchID string + Env string +} + +type DeleteAsyncSearchRequest struct { + SearchID string + Env string +} + +type DeleteExpiredAsyncSearchesRequest struct { + Env string } diff --git a/internal/app/types/ctx.go b/internal/app/types/ctx.go index b07e972..5570c5c 100644 --- a/internal/app/types/ctx.go +++ b/internal/app/types/ctx.go @@ -38,3 +38,18 @@ func GetUseSeqQL(ctx context.Context) string { } return useSeqQL } + +type EnvKey struct{} + +// SetEnvKey returns a new context with the env. +func SetEnvKey(ctx context.Context, env string) context.Context { + return context.WithValue(ctx, EnvKey{}, env) +} + +// GetEnvKey returns env from context. +func GetEnvKey(ctx context.Context) string { + if v := ctx.Value(EnvKey{}); v != nil { + return v.(string) + } + return "" +} diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index 445c390..0601dfc 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -4,51 +4,43 @@ import ( "context" "fmt" - "go.uber.org/zap" - - "github.com/ozontech/seq-ui/internal/app/config" - "github.com/ozontech/seq-ui/logger" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) -func NewRedis(ctx context.Context, cfg *config.Redis) (Cache, error) { - if cfg == nil { - return nil, nil - } - - return newRedisCache(ctx, cfg) -} +var caches map[string]Cache -func NewInmemoryWithRedisOrInmemory(ctx context.Context, cfg config.Cache) (Cache, error) { - return newRedisBasedOrInmemory(ctx, cfg, true) -} +func Init(ctx context.Context, cfg *config.Cache) error { + caches = make(map[string]Cache, len(cfg.Redis)+len(cfg.Inmemory)) -func NewRedisOrInmemory(ctx context.Context, cfg config.Cache) (Cache, error) { - return newRedisBasedOrInmemory(ctx, cfg, false) -} - -func newRedisBasedOrInmemory(ctx context.Context, cfg config.Cache, withInmem bool) (Cache, error) { - inmem, err := newInmemoryCache(cfg.Inmemory) - if err != nil { - return nil, fmt.Errorf("init inmemory cache: %w", err) + for i := range cfg.Inmemory { + inmemCfg := &cfg.Inmemory[i] + inmem, err := newInmemoryCache(inmemCfg) + if err != nil { + return fmt.Errorf("init inmemory cache %q: %w", inmemCfg.ID, err) + } + caches[inmemCfg.ID] = inmem } - if cfg.Redis == nil { - logger.Warn("redis cache config is nil; inmemory cache will be used instead") - return inmem, nil - } + for i := range cfg.Redis { + redisCfg := &cfg.Redis[i] - redis, err := newRedisCache(ctx, cfg.Redis) - if err != nil { - logger.Warn("failed to init redis cache; inmemory cache will be used instead", zap.Error(err)) - return inmem, nil - } + var inmem Cache + if redisCfg.WithInmemID != "" { + inmem = caches[redisCfg.WithInmemID] + } - if withInmem { - return &inmemWithRedis{ - inmem: inmem, - redis: redis, - }, nil + redis, err := newRedisCache(ctx, redisCfg, inmem) + if err != nil { + return fmt.Errorf("init redis cache %q: %w", redisCfg.ID, err) + } + caches[redisCfg.ID] = redis } - return redis, nil + return nil +} + +// ByID returns cache by id and true, false if cache is not found. +func ByID(id string) (Cache, bool) { + cache, ok := caches[id] + return cache, ok } diff --git a/internal/pkg/cache/inmemory.go b/internal/pkg/cache/inmemory.go index c407042..d58a9b0 100644 --- a/internal/pkg/cache/inmemory.go +++ b/internal/pkg/cache/inmemory.go @@ -7,7 +7,7 @@ import ( "github.com/dgraph-io/ristretto/v2" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/metric" ) @@ -17,7 +17,7 @@ type inmemoryCache struct { cache *ristretto.Cache[string, string] } -func newInmemoryCache(cfg config.InmemoryCache) (*inmemoryCache, error) { +func newInmemoryCache(cfg *config.Inmemory) (Cache, error) { c, err := ristretto.NewCache(&ristretto.Config[string, string]{ NumCounters: cfg.NumCounters, MaxCost: cfg.MaxCost, diff --git a/internal/pkg/cache/redis.go b/internal/pkg/cache/redis.go index 3982b9f..2676592 100644 --- a/internal/pkg/cache/redis.go +++ b/internal/pkg/cache/redis.go @@ -8,7 +8,7 @@ import ( "github.com/redis/go-redis/v9" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/redisclient" "github.com/ozontech/seq-ui/metric" ) @@ -18,15 +18,21 @@ type redisCache struct { prefix string } -func newRedisCache(ctx context.Context, cfg *config.Redis) (*redisCache, error) { +func newRedisCache(ctx context.Context, cfg *config.Redis, inmem Cache) (Cache, error) { client, err := redisclient.New(ctx, cfg) if err != nil { - metric.ServerCacheRedisMisses.Inc() return nil, fmt.Errorf("create redis client: %w", err) } - metric.ServerCacheRedisHits.Inc() - return &redisCache{client: client, prefix: cfg.KeyPrefix}, nil + rc := &redisCache{client: client, prefix: cfg.KeyPrefix} + if inmem == nil { + return rc, nil + } + + return &redisWithInmem{ + redis: rc, + inmem: inmem, + }, nil } func (c *redisCache) Set(ctx context.Context, key string, value string) error { @@ -40,10 +46,12 @@ func (c *redisCache) SetWithTTL(ctx context.Context, key string, value string, t func (c *redisCache) Get(ctx context.Context, key string) (string, error) { value, err := c.client.Get(ctx, c.withPrefix(key)).Result() if err == nil { + metric.ServerCacheRedisHits.Inc() return value, nil } if errors.Is(err, redis.Nil) { + metric.ServerCacheRedisMisses.Inc() return "", ErrNotFound } diff --git a/internal/pkg/cache/inmem_with_redis.go b/internal/pkg/cache/redis_with_inmem.go similarity index 76% rename from internal/pkg/cache/inmem_with_redis.go rename to internal/pkg/cache/redis_with_inmem.go index 304a984..f2422ac 100644 --- a/internal/pkg/cache/inmem_with_redis.go +++ b/internal/pkg/cache/redis_with_inmem.go @@ -10,16 +10,16 @@ import ( "github.com/ozontech/seq-ui/logger" ) -type inmemWithRedis struct { - inmem *inmemoryCache - redis *redisCache +type redisWithInmem struct { + redis Cache + inmem Cache } -func (c *inmemWithRedis) Set(ctx context.Context, key string, value string) error { +func (c *redisWithInmem) Set(ctx context.Context, key string, value string) error { return c.SetWithTTL(ctx, key, value, 0) } -func (c *inmemWithRedis) SetWithTTL(ctx context.Context, key string, value string, ttl time.Duration) error { +func (c *redisWithInmem) SetWithTTL(ctx context.Context, key string, value string, ttl time.Duration) error { if err := c.redis.SetWithTTL(ctx, key, value, ttl); err != nil { logger.Error("failed to set value in redis sub cache", zap.String("key", key), zap.Error(err)) } @@ -27,7 +27,7 @@ func (c *inmemWithRedis) SetWithTTL(ctx context.Context, key string, value strin return c.inmem.SetWithTTL(ctx, key, value, ttl) } -func (c *inmemWithRedis) Get(ctx context.Context, key string) (string, error) { +func (c *redisWithInmem) Get(ctx context.Context, key string) (string, error) { val, err := c.inmem.Get(ctx, key) if err == nil { return val, nil @@ -49,7 +49,7 @@ func (c *inmemWithRedis) Get(ctx context.Context, key string) (string, error) { return "", err } -func (c *inmemWithRedis) GetTTL(ctx context.Context, key string) (time.Duration, error) { +func (c *redisWithInmem) GetTTL(ctx context.Context, key string) (time.Duration, error) { val, err := c.inmem.GetTTL(ctx, key) if err == nil { return val, nil @@ -67,7 +67,7 @@ func (c *inmemWithRedis) GetTTL(ctx context.Context, key string) (time.Duration, return 0, err } -func (c *inmemWithRedis) Del(ctx context.Context, key string) { +func (c *redisWithInmem) Del(ctx context.Context, key string) { c.redis.Del(ctx, key) c.inmem.Del(ctx, key) } diff --git a/internal/pkg/client/seqdb/grpc_client.go b/internal/pkg/client/seqdb/grpc_client.go index 82c7082..0be1dbb 100644 --- a/internal/pkg/client/seqdb/grpc_client.go +++ b/internal/pkg/client/seqdb/grpc_client.go @@ -24,6 +24,8 @@ import ( "github.com/ozontech/seq-ui/metric" ) +const defaultMaxRecvMsgSize = 4 * 1024 * 1024 // 4MB + type grpcSearchResp interface { GetError() *seqproxyapi.Error } @@ -31,8 +33,8 @@ type grpcSearchResp interface { func processClientRequestUnaryInterceptor( ctx context.Context, method string, - req interface{}, - reply interface{}, + req any, + reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption, @@ -130,10 +132,6 @@ type GRPCClient struct { } func NewGRPCClient(ctx context.Context, params ClientParams) (*GRPCClient, error) { - if len(params.Addrs) == 0 { - panic("addrs is empty") - } - clients := make([]seqproxyapi.SeqProxyApiClient, 0, len(params.Addrs)) unaryInterceptors := []grpc.UnaryClientInterceptor{ processClientRequestUnaryInterceptor, @@ -147,7 +145,7 @@ func NewGRPCClient(ctx context.Context, params ClientParams) (*GRPCClient, error opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions( - grpc.MaxCallRecvMsgSize(params.MaxRecvMsgSize), + grpc.MaxCallRecvMsgSize(max(params.MaxRecvMsgSize, defaultMaxRecvMsgSize)), ), grpc.WithUnaryInterceptor(grpc_mw.ChainUnaryClient(unaryInterceptors...)), grpc.WithStreamInterceptor(grpc_mw.ChainStreamClient(streamInterceptors...)), diff --git a/internal/pkg/mask/field_filter.go b/internal/pkg/mask/field_filter.go index 2eba3dc..156077c 100644 --- a/internal/pkg/mask/field_filter.go +++ b/internal/pkg/mask/field_filter.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) type filterCondition int8 diff --git a/internal/pkg/mask/field_filter_test.go b/internal/pkg/mask/field_filter_test.go index 4c368bc..885dec4 100644 --- a/internal/pkg/mask/field_filter_test.go +++ b/internal/pkg/mask/field_filter_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestFieldFilterSetCreate(t *testing.T) { diff --git a/internal/pkg/mask/mask.go b/internal/pkg/mask/mask.go index f3d4000..523c096 100644 --- a/internal/pkg/mask/mask.go +++ b/internal/pkg/mask/mask.go @@ -7,7 +7,7 @@ import ( "slices" "strings" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) const maskSymbol = byte('*') diff --git a/internal/pkg/mask/mask_test.go b/internal/pkg/mask/mask_test.go index afe054e..152c4fe 100644 --- a/internal/pkg/mask/mask_test.go +++ b/internal/pkg/mask/mask_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestMaskCompile(t *testing.T) { diff --git a/internal/pkg/mask/masker.go b/internal/pkg/mask/masker.go index 90adb62..4b1185f 100644 --- a/internal/pkg/mask/masker.go +++ b/internal/pkg/mask/masker.go @@ -3,7 +3,7 @@ package mask import ( "fmt" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) type Masker struct { diff --git a/internal/pkg/mask/masker_test.go b/internal/pkg/mask/masker_test.go index b8da9d2..c7b64f4 100644 --- a/internal/pkg/mask/masker_test.go +++ b/internal/pkg/mask/masker_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestMaskerMask(t *testing.T) { diff --git a/internal/pkg/redisclient/client.go b/internal/pkg/redisclient/client.go index e9a879b..bd4edb8 100644 --- a/internal/pkg/redisclient/client.go +++ b/internal/pkg/redisclient/client.go @@ -6,7 +6,7 @@ import ( "github.com/redis/go-redis/v9" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) func New(ctx context.Context, cfg *config.Redis) (*redis.Client, error) { diff --git a/internal/pkg/repository/admin_repo.go b/internal/pkg/repository/admin_repo.go index 0b6eecf..12b48f3 100644 --- a/internal/pkg/repository/admin_repo.go +++ b/internal/pkg/repository/admin_repo.go @@ -45,9 +45,9 @@ func (r *adminRepository) CreateRole(ctx context.Context, req types.CreateRoleRe func (r *adminRepository) AddUsersToRole(ctx context.Context, req types.AddUsersToRoleRequest) error { metricLabels := []string{"users_roles", "INSERT"} - query, args := `INSERT INTO users_roles (user_id, role_id) - SELECT id, $1 FROM user_profiles WHERE user_name=ANY($2) - ON CONFLICT DO NOTHING`, []any{req.RoleID, req.Usernames} + query, args := `INSERT INTO users_roles (user_id, role_id, env) + SELECT id, $1, $2 FROM user_profiles WHERE user_name=ANY($3) + ON CONFLICT DO NOTHING`, []any{req.RoleID, req.Env, req.Usernames} if _, err := r.pool.exec(ctx, metricLabels, query, args...); err != nil { incErrorMetric(err, metricLabels) @@ -59,8 +59,8 @@ func (r *adminRepository) AddUsersToRole(ctx context.Context, req types.AddUsers func (r *adminRepository) DeleteUsersFromRole(ctx context.Context, req types.DeleteUsersFromRoleRequest) error { metricLabels := []string{"users_roles", "DELETE"} - query, args := `DELETE FROM users_roles WHERE role_id=$1 AND user_id IN ( - SELECT id FROM user_profiles WHERE user_name=ANY($2))`, []any{req.RoleID, req.Usernames} + query, args := `DELETE FROM users_roles WHERE role_id=$1 AND env=$2 AND user_id IN ( + SELECT id FROM user_profiles WHERE user_name=ANY($3))`, []any{req.RoleID, req.Env, req.Usernames} if _, err := r.pool.exec(ctx, metricLabels, query, args...); err != nil { incErrorMetric(err, metricLabels) @@ -100,7 +100,7 @@ func (r *adminRepository) GetRoles(ctx context.Context) ([]types.Role, error) { func (r *adminRepository) GetRole(ctx context.Context, req types.GetRoleRequest) (types.RoleInfo, error) { metricLabels := []string{"user_profiles", "SELECT"} query, args := `SELECT up.user_name FROM user_profiles up JOIN users_roles ur ON up.id = ur.user_id - WHERE ur.role_id=$1 ORDER BY up.user_name`, []any{req.RoleID} + WHERE ur.role_id=$1 AND ur.env=$2 ORDER BY up.user_name`, []any{req.RoleID, req.Env} rows, err := r.pool.query(ctx, metricLabels, query, args...) if err != nil { @@ -162,8 +162,8 @@ func (r *adminRepository) DeleteRole(ctx context.Context, req types.DeleteRoleRe return r.txManager.Do(ctx, func(tx pgx.Tx) error { if req.ReplacementRoleID != nil { metricLabels := []string{"users_roles", "INSERT"} - query, args := `INSERT INTO users_roles (user_id, role_id) - SELECT user_id, $1 FROM users_roles WHERE role_id=$2 ON CONFLICT DO NOTHING`, []any{*req.ReplacementRoleID, req.RoleID} + query, args := `INSERT INTO users_roles (user_id, role_id, env) + SELECT user_id, $1, env FROM users_roles WHERE role_id=$2 ON CONFLICT DO NOTHING`, []any{*req.ReplacementRoleID, req.RoleID} if _, err := r.pool.execTx(ctx, tx, metricLabels, query, args...); err != nil { incErrorMetric(err, metricLabels) @@ -209,7 +209,7 @@ func (r *adminRepository) GetUserPermissions(ctx context.Context, req types.GetU JOIN users_roles ur ON up.id=ur.user_id JOIN roles_permissions rp ON ur.role_id=rp.role_id JOIN permissions p ON rp.permission_id=p.id - WHERE up.user_name=$1 ORDER BY p.value`, []any{req.Username} + WHERE ur.env=$1 up.user_name=$2 ORDER BY p.value`, []any{req.Env, req.Username} rows, err := r.pool.query(ctx, metricLabels, query, args...) if err != nil { diff --git a/internal/pkg/repository/async_searches_repo.go b/internal/pkg/repository/async_searches_repo.go index 8594134..fcfc99d 100644 --- a/internal/pkg/repository/async_searches_repo.go +++ b/internal/pkg/repository/async_searches_repo.go @@ -20,12 +20,9 @@ func newAsyncSearchesRepository(pool *pool) *asyncSearchesRepository { return &asyncSearchesRepository{pool} } -func (r *asyncSearchesRepository) SaveAsyncSearch( - ctx context.Context, - req types.SaveAsyncSearchRequest, -) error { - query, args := "INSERT INTO async_searches (search_id,owner_id,meta,expires_at) VALUES ($1,$2,$3,$4)", - []any{req.SearchID, req.OwnerID, req.Meta, req.ExpiresAt} +func (r *asyncSearchesRepository) SaveAsyncSearch(ctx context.Context, req types.SaveAsyncSearchRequest) error { + query, args := "INSERT INTO async_searches (search_id,owner_id,meta,expires_at,env) VALUES ($1,$2,$3,$4,$5)", + []any{req.SearchID, req.OwnerID, req.Meta, req.ExpiresAt, req.Env} metricLabels := []string{"async_searches", "INSERT"} _, err := r.exec(ctx, metricLabels, query, args...) @@ -37,19 +34,16 @@ func (r *asyncSearchesRepository) SaveAsyncSearch( return nil } -func (r *asyncSearchesRepository) GetAsyncSearchById( - ctx context.Context, - searchID string, -) (types.AsyncSearchInfo, error) { +func (r *asyncSearchesRepository) GetAsyncSearchById(ctx context.Context, req types.GetAsyncSearchByIdRequest) (types.AsyncSearchInfo, error) { as := types.AsyncSearchInfo{} query, args := ` SELECT s.search_id, s.owner_id, p.user_name, s.meta, s.created_at, s.expires_at FROM async_searches AS s JOIN user_profiles AS p ON p.id = s.owner_id - WHERE s.search_id = $1 + WHERE s.search_id = $1 AND s.env = $2 `, - []any{searchID} + []any{req.SearchID, req.Env} metricLabels := []string{"async_searches", "SELECT"} err := r.queryRow(ctx, metricLabels, query, args...).Scan( @@ -74,13 +68,11 @@ func (r *asyncSearchesRepository) GetAsyncSearchById( return as, nil } -func (r *asyncSearchesRepository) GetAsyncSearchesList( - ctx context.Context, - req types.GetAsyncSearchesListRequest, -) ([]types.AsyncSearchInfo, error) { +func (r *asyncSearchesRepository) GetAsyncSearchesList(ctx context.Context, req types.GetAsyncSearchesListRequest) ([]types.AsyncSearchInfo, error) { qb := sqlb.Select("s.search_id", "s.owner_id", "p.user_name", "s.created_at", "s.expires_at"). From("async_searches AS s"). Join("user_profiles AS p ON p.id = s.owner_id"). + Where(sq.Eq{"s.env": req.Env}). Where(sq.GtOrEq{"s.expires_at": "now()"}). OrderBy("s.created_at DESC") @@ -119,12 +111,9 @@ func (r *asyncSearchesRepository) GetAsyncSearchesList( return asyncSearches, nil } -func (r *asyncSearchesRepository) DeleteAsyncSearch( - ctx context.Context, - searchID string, -) error { - query, args := "DELETE FROM async_searches WHERE search_id = $1", - []any{searchID} +func (r *asyncSearchesRepository) DeleteAsyncSearch(ctx context.Context, req types.DeleteAsyncSearchRequest) error { + query, args := "DELETE FROM async_searches WHERE search_id = $1 AND env = $2", + []any{req.SearchID, req.Env} metricLabelsDelete := []string{"async_searches", "DELETE"} if _, err := r.exec(ctx, metricLabelsDelete, query, args...); err != nil { @@ -135,11 +124,12 @@ func (r *asyncSearchesRepository) DeleteAsyncSearch( return nil } -func (r *asyncSearchesRepository) DeleteExpiredAsyncSearches(ctx context.Context) error { - query := "DELETE FROM async_searches WHERE expires_at < now()" +func (r *asyncSearchesRepository) DeleteExpiredAsyncSearches(ctx context.Context, req types.DeleteExpiredAsyncSearchesRequest) error { + query, args := "DELETE FROM async_searches WHERE env = $1 AND expires_at < now()", + []any{req.Env} metricLabelsDelete := []string{"async_searches", "DELETE"} - if _, err := r.exec(ctx, metricLabelsDelete, query); err != nil { + if _, err := r.exec(ctx, metricLabelsDelete, query, args...); err != nil { incErrorMetric(err, metricLabelsDelete) return fmt.Errorf("failed to delete expired async searches: %w", err) } diff --git a/internal/pkg/repository/mock/repository.go b/internal/pkg/repository/mock/repository.go index 8efcc30..5e3004d 100644 --- a/internal/pkg/repository/mock/repository.go +++ b/internal/pkg/repository/mock/repository.go @@ -290,7 +290,7 @@ func (m *MockAsyncSearches) EXPECT() *MockAsyncSearchesMockRecorder { } // DeleteAsyncSearch mocks base method. -func (m *MockAsyncSearches) DeleteAsyncSearch(arg0 context.Context, arg1 string) error { +func (m *MockAsyncSearches) DeleteAsyncSearch(arg0 context.Context, arg1 types.DeleteAsyncSearchRequest) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteAsyncSearch", arg0, arg1) ret0, _ := ret[0].(error) @@ -304,21 +304,21 @@ func (mr *MockAsyncSearchesMockRecorder) DeleteAsyncSearch(arg0, arg1 any) *gomo } // DeleteExpiredAsyncSearches mocks base method. -func (m *MockAsyncSearches) DeleteExpiredAsyncSearches(arg0 context.Context) error { +func (m *MockAsyncSearches) DeleteExpiredAsyncSearches(arg0 context.Context, arg1 types.DeleteExpiredAsyncSearchesRequest) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteExpiredAsyncSearches", arg0) + ret := m.ctrl.Call(m, "DeleteExpiredAsyncSearches", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // DeleteExpiredAsyncSearches indicates an expected call of DeleteExpiredAsyncSearches. -func (mr *MockAsyncSearchesMockRecorder) DeleteExpiredAsyncSearches(arg0 any) *gomock.Call { +func (mr *MockAsyncSearchesMockRecorder) DeleteExpiredAsyncSearches(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteExpiredAsyncSearches", reflect.TypeOf((*MockAsyncSearches)(nil).DeleteExpiredAsyncSearches), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteExpiredAsyncSearches", reflect.TypeOf((*MockAsyncSearches)(nil).DeleteExpiredAsyncSearches), arg0, arg1) } // GetAsyncSearchById mocks base method. -func (m *MockAsyncSearches) GetAsyncSearchById(arg0 context.Context, arg1 string) (types.AsyncSearchInfo, error) { +func (m *MockAsyncSearches) GetAsyncSearchById(arg0 context.Context, arg1 types.GetAsyncSearchByIdRequest) (types.AsyncSearchInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAsyncSearchById", arg0, arg1) ret0, _ := ret[0].(types.AsyncSearchInfo) diff --git a/internal/pkg/repository/repository.go b/internal/pkg/repository/repository.go index 6087d20..d425960 100644 --- a/internal/pkg/repository/repository.go +++ b/internal/pkg/repository/repository.go @@ -33,9 +33,9 @@ type ( AsyncSearches interface { SaveAsyncSearch(context.Context, types.SaveAsyncSearchRequest) error - GetAsyncSearchById(context.Context, string) (types.AsyncSearchInfo, error) - DeleteAsyncSearch(context.Context, string) error - DeleteExpiredAsyncSearches(context.Context) error + GetAsyncSearchById(context.Context, types.GetAsyncSearchByIdRequest) (types.AsyncSearchInfo, error) + DeleteAsyncSearch(context.Context, types.DeleteAsyncSearchRequest) error + DeleteExpiredAsyncSearches(context.Context, types.DeleteExpiredAsyncSearchesRequest) error GetAsyncSearchesList(context.Context, types.GetAsyncSearchesListRequest) ([]types.AsyncSearchInfo, error) } diff --git a/internal/pkg/service/admin/cache.go b/internal/pkg/service/admin/cache.go index cb9ebba..5fb9035 100644 --- a/internal/pkg/service/admin/cache.go +++ b/internal/pkg/service/admin/cache.go @@ -22,12 +22,18 @@ const ( var errCacheDisabled = errors.New("admin cache disabled") type adminCache struct { - cache cache.Cache - ttl time.Duration + cache cache.Cache + userPermsKeyPrefix string + ttl time.Duration } -func newAdminCache(c cache.Cache, ttl time.Duration) adminCache { - return adminCache{cache: c, ttl: ttl} +func newAdminCache(c cache.Cache, ttl time.Duration, env string) adminCache { + userPermsKeyPrefix := cacheKeyUserPerms + if env != "" { + userPermsKeyPrefix += env + "_" + } + + return adminCache{cache: c, ttl: ttl, userPermsKeyPrefix: userPermsKeyPrefix} } func (c *adminCache) getRoles(ctx context.Context) ([]types.Role, error) { @@ -77,7 +83,7 @@ func (c *adminCache) getUserPermissions(ctx context.Context, username string) ([ return nil, errCacheDisabled } - data, err := c.cache.Get(ctx, cacheKeyUserPerms+username) + data, err := c.cache.Get(ctx, c.userPermsKeyPrefix+username) if err != nil { return nil, fmt.Errorf("failed to get user permissions from cache: %w", err) } @@ -101,7 +107,7 @@ func (c *adminCache) setUserPermissions(ctx context.Context, username string, pe return } - if err := c.cache.SetWithTTL(ctx, cacheKeyUserPerms+username, string(data), c.ttl); err != nil { + if err := c.cache.SetWithTTL(ctx, c.userPermsKeyPrefix+username, string(data), c.ttl); err != nil { logger.Error("failed to set user permissions in cache", zap.Error(err)) } } @@ -112,6 +118,6 @@ func (c *adminCache) resetUsersPermissions(ctx context.Context, usernames ...str } for _, username := range usernames { - c.cache.Del(ctx, cacheKeyUserPerms+username) + c.cache.Del(ctx, c.userPermsKeyPrefix+username) } } diff --git a/internal/pkg/service/admin/permissions.go b/internal/pkg/service/admin/permissions.go index 5d3eb15..3d27db2 100644 --- a/internal/pkg/service/admin/permissions.go +++ b/internal/pkg/service/admin/permissions.go @@ -34,7 +34,10 @@ func (s *service) checkAccess(ctx context.Context, requiredPermission string) er return nil } - permissions, err := s.getUserPermissions(ctx, types.GetUserPermissionsRequest{Username: username}) + permissions, err := s.getUserPermissions(ctx, types.GetUserPermissionsRequest{ + Username: username, + Env: s.env, + }) if err != nil { return fmt.Errorf("can't get user permissions: %w", err) } diff --git a/internal/pkg/service/admin/permissions_test.go b/internal/pkg/service/admin/permissions_test.go index a0922de..5294654 100644 --- a/internal/pkg/service/admin/permissions_test.go +++ b/internal/pkg/service/admin/permissions_test.go @@ -68,7 +68,7 @@ func TestCheckAccess(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg).(*service) + svc := New(adminRepo, upRepo, cache, adminCfg, "").(*service) if tt.mockArgs != nil { cache.EXPECT(). @@ -80,7 +80,7 @@ func TestCheckAccess(t *testing.T) { Return(tt.mockArgs.perms, tt.mockArgs.err). Times(1) cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.CacheTTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -152,7 +152,7 @@ func TestGetUserPermissions(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg).(*service) + svc := New(adminRepo, upRepo, cache, adminCfg, "").(*service) cache.EXPECT(). Get(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username). @@ -167,7 +167,7 @@ func TestGetUserPermissions(t *testing.T) { if tt.mockArgs.err == nil { cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.CacheTTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+tt.mockArgs.req.Username, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -220,7 +220,7 @@ func TestValidatePermissions(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg).(*service) + svc := New(adminRepo, upRepo, cache, adminCfg, "").(*service) err := svc.validatePermissions(tt.perms) require.Equal(t, tt.wantErr, err != nil) diff --git a/internal/pkg/service/admin/roles.go b/internal/pkg/service/admin/roles.go index 2b5192c..83bea1b 100644 --- a/internal/pkg/service/admin/roles.go +++ b/internal/pkg/service/admin/roles.go @@ -60,6 +60,7 @@ func (s *service) AddUsersToRole(ctx context.Context, req types.AddUsersToRoleRe } } + req.Env = s.env if err := s.adminRepo.AddUsersToRole(ctx, req); err != nil { return err } @@ -86,6 +87,7 @@ func (s *service) DeleteUsersFromRole(ctx context.Context, req types.DeleteUsers return types.NewErrInvalidRequestField("empty username") } + req.Env = s.env if err := s.adminRepo.DeleteUsersFromRole(ctx, req); err != nil { return err } @@ -125,6 +127,7 @@ func (s *service) GetRole(ctx context.Context, req types.GetRoleRequest) (types. return types.RoleInfo{}, types.NewErrInvalidRequestField("value role_id must be greater than 0") } + req.Env = s.env return s.adminRepo.GetRole(ctx, req) } @@ -154,7 +157,7 @@ func (s *service) UpdateRole(ctx context.Context, req types.UpdateRoleRequest) e s.cache.resetRoles(ctx) if len(req.Permissions) > 0 { - if roleInfo, err := s.adminRepo.GetRole(ctx, types.GetRoleRequest{RoleID: req.RoleID}); err == nil { + if roleInfo, err := s.adminRepo.GetRole(ctx, types.GetRoleRequest{RoleID: req.RoleID, Env: s.env}); err == nil { s.cache.resetUsersPermissions(ctx, roleInfo.Usernames...) } } @@ -180,7 +183,7 @@ func (s *service) DeleteRole(ctx context.Context, req types.DeleteRoleRequest) e } } - if roleInfo, err := s.adminRepo.GetRole(ctx, types.GetRoleRequest{RoleID: req.RoleID}); err == nil { + if roleInfo, err := s.adminRepo.GetRole(ctx, types.GetRoleRequest{RoleID: req.RoleID, Env: s.env}); err == nil { s.cache.resetUsersPermissions(ctx, roleInfo.Usernames...) } diff --git a/internal/pkg/service/admin/roles_test.go b/internal/pkg/service/admin/roles_test.go index f280f38..babfcf5 100644 --- a/internal/pkg/service/admin/roles_test.go +++ b/internal/pkg/service/admin/roles_test.go @@ -9,15 +9,20 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock "github.com/ozontech/seq-ui/internal/pkg/repository/mock" ) var ( - defaultSuperUser = "superuser" - adminCfg = &config.Admin{SuperUsers: []string{defaultSuperUser}, CacheTTL: 5 * time.Minute} + defaultSuperUser = "superuser" + adminCfg = &config.AdminOptions{ + SuperUsers: []string{defaultSuperUser}, + Cache: &config.HandlerCache{ + TTL: 5 * time.Minute, + }, + } errSomethingWrong = errors.New("something happened wrong") ) @@ -46,7 +51,7 @@ func setupAccessMock(ctx context.Context, repo *mock.MockAdmin, cache *mock_cach Times(1) cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyUserPerms+access.username, gomock.Any(), adminCfg.CacheTTL). + SetWithTTL(gomock.Any(), cacheKeyUserPerms+access.username, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -193,7 +198,7 @@ func TestCreateRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -340,7 +345,7 @@ func TestAddUsersToRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -458,7 +463,7 @@ func TestGetRoles(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -475,7 +480,7 @@ func TestGetRoles(t *testing.T) { if tt.mockArgs.err == nil { cache.EXPECT(). - SetWithTTL(gomock.Any(), cacheKeyRoles, gomock.Any(), adminCfg.CacheTTL). + SetWithTTL(gomock.Any(), cacheKeyRoles, gomock.Any(), adminCfg.Cache.TTL). Return(nil). Times(1) } @@ -570,7 +575,7 @@ func TestGetRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -781,7 +786,7 @@ func TestUpdateRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -939,7 +944,7 @@ func TestDeleteRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) @@ -1095,7 +1100,7 @@ func TestDeleteUsersFromRole(t *testing.T) { adminRepo := mock.NewMockAdmin(ctrl) upRepo := mock.NewMockUserProfiles(ctrl) cache := mock_cache.NewMockCache(ctrl) - svc := New(adminRepo, upRepo, cache, adminCfg) + svc := New(adminRepo, upRepo, cache, adminCfg, "") ctx := setupAccessMock(context.Background(), adminRepo, cache, tt.accessMock) diff --git a/internal/pkg/service/admin/service.go b/internal/pkg/service/admin/service.go index de94af4..9d7d9e8 100644 --- a/internal/pkg/service/admin/service.go +++ b/internal/pkg/service/admin/service.go @@ -2,8 +2,9 @@ package admin import ( "context" + "time" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/repository" @@ -25,18 +26,46 @@ type service struct { userProfilesRepo repository.UserProfiles cache adminCache superUsers map[string]struct{} + env string } -func New(adminRepo repository.Admin, userProfilesRepo repository.UserProfiles, c cache.Cache, cfg *config.Admin) Service { +func New(adminRepo repository.Admin, userProfilesRepo repository.UserProfiles, c cache.Cache, cfg *config.AdminOptions, env string) Service { su := make(map[string]struct{}, len(cfg.SuperUsers)) for _, u := range cfg.SuperUsers { su[u] = struct{}{} } + var ttl time.Duration + if cfg.Cache != nil { + ttl = cfg.Cache.TTL + } + return &service{ adminRepo: adminRepo, userProfilesRepo: userProfilesRepo, - cache: newAdminCache(c, cfg.CacheTTL), + cache: newAdminCache(c, ttl, env), superUsers: su, + env: env, + } +} + +func NewByEnv(cfg *config.Admin, adminRepo repository.Admin, userProfilesRepo repository.UserProfiles) map[string]Service { + services := make(map[string]Service, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envCfg := range cfg.Envs { + var c cache.Cache + if envCfg.Options.Cache != nil { + c, _ = cache.ByID(envCfg.Options.Cache.ID) + } + services[envName] = New(adminRepo, userProfilesRepo, c, envCfg.Options, envName) + } + } else { + var c cache.Cache + if cfg.Options.Cache != nil { + c, _ = cache.ByID(cfg.Options.Cache.ID) + } + services[""] = New(adminRepo, userProfilesRepo, c, &cfg.Options, "") } + + return services } diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index a772afa..8af7bb0 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -11,7 +11,7 @@ import ( "go.uber.org/zap" "github.com/ozontech/seq-ui/internal/api/httputil" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/repository" @@ -41,14 +41,16 @@ type service struct { repo repository.AsyncSearches seqDB seqdb.Client - cfg config.AsyncSearch + cfg *config.AsyncSearchOptions + env string } -func New(ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, cfg config.AsyncSearch) Service { +func New(ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, cfg *config.AsyncSearchOptions, env string) Service { s := &service{ repo: repo, seqDB: seqDB, cfg: cfg, + env: env, } go s.deleteExpiredAsyncSearches(ctx) @@ -56,6 +58,33 @@ func New(ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, return s } +// NewByEnv creates service for each env. +func NewByEnv( + ctx context.Context, + cfg *config.AsyncSearch, + repo repository.AsyncSearches, + seqDBClients map[string]seqdb.Client, +) map[string]Service { + if cfg == nil || repo == nil { + return nil + } + + services := make(map[string]Service, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envCfg := range cfg.Envs { + services[envName] = New(ctx, repo, seqDBClients[envCfg.SeqDBID], envCfg.Options, envName) + } + } else { + var client seqdb.Client + for _, c := range seqDBClients { + client = c + } + services[""] = New(ctx, repo, client, &cfg.Options, "") + } + + return services +} + func (s *service) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSearchRequest) (*seqapi.StartAsyncSearchResponse, error) { ownerID, err := profiles.GetIDFromContext(ctx) if err != nil { @@ -77,6 +106,7 @@ func (s *service) StartAsyncSearch(ctx context.Context, req *seqapi.StartAsyncSe OwnerID: ownerID, ExpiresAt: time.Now().Add(req.Retention.AsDuration()), Meta: req.Meta, + Env: s.env, }) }, backoff.NewExponentialBackOff( backoff.WithInitialInterval(backoffInitialInterval), @@ -95,7 +125,10 @@ func (s *service) DeleteAsyncSearch(ctx context.Context, req *seqapi.DeleteAsync return nil, err } - searchInfo, err := s.repo.GetAsyncSearchById(ctx, req.SearchId) + searchInfo, err := s.repo.GetAsyncSearchById(ctx, types.GetAsyncSearchByIdRequest{ + SearchID: req.SearchId, + Env: s.env, + }) if err != nil { return nil, fmt.Errorf("failed to get async search by id: %w", err) } @@ -111,7 +144,10 @@ func (s *service) DeleteAsyncSearch(ctx context.Context, req *seqapi.DeleteAsync // we can retry multiple times after the async search is deleted from seq-db err = backoff.Retry(func() error { - return s.repo.DeleteAsyncSearch(ctx, req.SearchId) + return s.repo.DeleteAsyncSearch(ctx, types.DeleteAsyncSearchRequest{ + SearchID: req.SearchId, + Env: s.env, + }) }, backoff.NewExponentialBackOff( backoff.WithInitialInterval(backoffInitialInterval), backoff.WithMaxElapsedTime(maxBackoffElapsedTime), @@ -129,7 +165,10 @@ func (s *service) CancelAsyncSearch(ctx context.Context, req *seqapi.CancelAsync return nil, err } - searchInfo, err := s.repo.GetAsyncSearchById(ctx, req.SearchId) + searchInfo, err := s.repo.GetAsyncSearchById(ctx, types.GetAsyncSearchByIdRequest{ + SearchID: req.SearchId, + Env: s.env, + }) if err != nil { return nil, fmt.Errorf("failed to get async search by id: %w", err) } @@ -147,7 +186,10 @@ func (s *service) CancelAsyncSearch(ctx context.Context, req *seqapi.CancelAsync } func (s *service) FetchAsyncSearchResult(ctx context.Context, req *seqapi.FetchAsyncSearchResultRequest) (*seqapi.FetchAsyncSearchResultResponse, error) { - searchInfo, err := s.repo.GetAsyncSearchById(ctx, req.SearchId) + searchInfo, err := s.repo.GetAsyncSearchById(ctx, types.GetAsyncSearchByIdRequest{ + SearchID: req.SearchId, + Env: s.env, + }) if err != nil { return nil, fmt.Errorf("failed to get async search by id: %w", err) } @@ -172,6 +214,7 @@ func (s *service) ExportAsyncSearch(ctx context.Context, req *seqapi.ExportAsync func (s *service) GetAsyncSearchesList(ctx context.Context, req *seqapi.GetAsyncSearchesListRequest) (*seqapi.GetAsyncSearchesListResponse, error) { searches, err := s.repo.GetAsyncSearchesList(ctx, types.GetAsyncSearchesListRequest{ Owner: req.OwnerName, + Env: s.env, }) if err != nil { return nil, fmt.Errorf("failed to get async searches list from db: %w", err) @@ -219,7 +262,9 @@ func (s *service) deleteExpiredAsyncSearches(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - err := s.repo.DeleteExpiredAsyncSearches(ctx) + err := s.repo.DeleteExpiredAsyncSearches(ctx, types.DeleteExpiredAsyncSearchesRequest{ + Env: s.env, + }) if err != nil { logger.Error("DeleteExpiredAsyncSearches error", zap.Error(err)) } diff --git a/internal/pkg/service/errorgroups/service.go b/internal/pkg/service/errorgroups/service.go index 8846ba0..d7fd96e 100644 --- a/internal/pkg/service/errorgroups/service.go +++ b/internal/pkg/service/errorgroups/service.go @@ -7,9 +7,10 @@ import ( "math" "slices" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "golang.org/x/sync/errgroup" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" repositorych "github.com/ozontech/seq-ui/internal/pkg/repository_ch" ) @@ -46,6 +47,25 @@ func New(repo repositorych.Repository, logTagsMapping config.LogTagsMapping) Ser } } +func NewByEnv(cfg *config.ErrorGroups, clientsCfg *config.Clients, chClients map[string]driver.Conn) map[string]Service { + services := make(map[string]Service, max(1, len(cfg.Envs))) + if len(cfg.Envs) > 0 { + for envName, envCfg := range cfg.Envs { + repo := repositorych.New(chClients[envCfg.CHID], clientsCfg.ClickHouseByID(envCfg.CHID).Sharded, envCfg.Options.QueryFilter) + services[envName] = New(repo, envCfg.Options.LogTagsMapping) + } + } else { + var chID string + for id := range chClients { + chID = id + } + repo := repositorych.New(chClients[chID], clientsCfg.ClickHouseByID(chID).Sharded, cfg.Options.QueryFilter) + services[""] = New(repo, cfg.Options.LogTagsMapping) + } + + return services +} + func (s *service) GetErrorGroups( ctx context.Context, req types.GetErrorGroupsRequest, diff --git a/internal/pkg/service/errorgroups/service_test.go b/internal/pkg/service/errorgroups/service_test.go index ef5c094..b9bd1ab 100644 --- a/internal/pkg/service/errorgroups/service_test.go +++ b/internal/pkg/service/errorgroups/service_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" mock "github.com/ozontech/seq-ui/internal/pkg/repository_ch/mock" ) diff --git a/internal/pkg/service/massexport/filestore/s3.go b/internal/pkg/service/massexport/filestore/s3.go index 48ef057..3b5309e 100644 --- a/internal/pkg/service/massexport/filestore/s3.go +++ b/internal/pkg/service/massexport/filestore/s3.go @@ -11,7 +11,7 @@ import ( "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" ) type s3FileStore struct { @@ -22,7 +22,7 @@ type s3FileStore struct { func NewS3(ctx context.Context, cfg *config.S3) (FileStore, error) { scheme := "http" - if cfg.EnableSSl { + if cfg.EnableSSL { scheme = "https" } diff --git a/internal/pkg/service/massexport/seq_proxy_downloader.go b/internal/pkg/service/massexport/seq_proxy_downloader.go index 3ff9163..5823e14 100644 --- a/internal/pkg/service/massexport/seq_proxy_downloader.go +++ b/internal/pkg/service/massexport/seq_proxy_downloader.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/metric" @@ -17,13 +17,13 @@ import ( ) type seqProxyDownloader struct { - cfg config.SeqProxyDownloader + cfg config.DownloadParams client seqdb.Client } const defaultDelay = 1 * time.Second -func newSeqProxyDownloader(client seqdb.Client, cfg config.SeqProxyDownloader) *seqProxyDownloader { +func newSeqProxyDownloader(client seqdb.Client, cfg config.DownloadParams) *seqProxyDownloader { if cfg.Delay <= 0 { cfg.Delay = defaultDelay } diff --git a/internal/pkg/service/massexport/service.go b/internal/pkg/service/massexport/service.go index 3c21309..ee62510 100644 --- a/internal/pkg/service/massexport/service.go +++ b/internal/pkg/service/massexport/service.go @@ -7,7 +7,7 @@ import ( "slices" "time" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/service/massexport/filestore" @@ -93,7 +93,7 @@ func NewService( tasksChannelSize = cfg.TasksChannelSize } - downloader := newSeqProxyDownloader(client, *cfg.SeqProxyDownloader) + downloader := newSeqProxyDownloader(client, *cfg.DownloadParams) return &exportService{ sessionStore: sessionStore, diff --git a/internal/pkg/service/massexport/sessionstore/redis.go b/internal/pkg/service/massexport/sessionstore/redis.go index f24d9b1..c4b245f 100644 --- a/internal/pkg/service/massexport/sessionstore/redis.go +++ b/internal/pkg/service/massexport/sessionstore/redis.go @@ -11,7 +11,7 @@ import ( "github.com/redis/go-redis/v9" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/redisclient" "github.com/ozontech/seq-ui/logger" @@ -31,23 +31,23 @@ const ( maxLifetime = 30 * day ) -func NewRedisSessionStore(ctx context.Context, cfg *config.SessionStore) (SessionStore, error) { - client, err := redisclient.New(ctx, &cfg.Redis) +func NewRedisSessionStore(ctx context.Context, redisCfg *config.Redis, exportLifetimeCfg time.Duration) (SessionStore, error) { + client, err := redisclient.New(ctx, redisCfg) if err != nil { return nil, fmt.Errorf("create redis client: %w", err) } exportLifetime := defaultLifetime - if cfg.ExportLifetime != 0 { + if exportLifetimeCfg != 0 { switch { - case cfg.ExportLifetime < minLifetime: - logger.Warn("export lifetime from config is too low; set min lifetime", zap.String("export_lifetime", cfg.ExportLifetime.String())) + case exportLifetimeCfg < minLifetime: + logger.Warn("export lifetime from config is too low; set min lifetime", zap.String("export_lifetime", exportLifetimeCfg.String())) exportLifetime = minLifetime - case cfg.ExportLifetime > maxLifetime: - logger.Warn("export lifetime from config is too high; set max lifetime", zap.String("export_lifetime", cfg.ExportLifetime.String())) + case exportLifetimeCfg > maxLifetime: + logger.Warn("export lifetime from config is too high; set max lifetime", zap.String("export_lifetime", exportLifetimeCfg.String())) exportLifetime = maxLifetime default: - exportLifetime = cfg.ExportLifetime + exportLifetime = exportLifetimeCfg } } diff --git a/internal/pkg/service/profiles/profiles.go b/internal/pkg/service/profiles/profiles.go index a473a97..0b8c815 100644 --- a/internal/pkg/service/profiles/profiles.go +++ b/internal/pkg/service/profiles/profiles.go @@ -11,7 +11,7 @@ type getOrCreateFn func(context.Context, types.GetOrCreateUserProfileRequest) (t var profile *profiles -func InitProfiles(fn getOrCreateFn) { +func Init(fn getOrCreateFn) { profile = &profiles{ idByName: make(map[string]int64), getOrCreateFn: fn, diff --git a/migration/13_add_column_env.sql b/migration/13_add_column_env.sql new file mode 100644 index 0000000..838482c --- /dev/null +++ b/migration/13_add_column_env.sql @@ -0,0 +1,27 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE IF EXISTS async_searches ADD COLUMN IF NOT EXISTS env text NOT NULL DEFAULT ''; +ALTER TABLE IF EXISTS async_searches DROP CONSTRAINT IF EXISTS async_searches_pkey; +ALTER TABLE IF EXISTS async_searches ADD PRIMARY KEY (env, search_id); + +ALTER TABLE IF EXISTS users_roles ADD COLUMN IF NOT EXISTS env text NOT NULL DEFAULT ''; +ALTER TABLE IF EXISTS users_roles DROP CONSTRAINT IF EXISTS users_roles_pkey; +ALTER TABLE IF EXISTS users_roles ADD PRIMARY KEY (user_id, env, role_id); + +DROP INDEX IF EXISTS idx_users_roles_role_id; +CREATE INDEX IF NOT EXISTS idx_users_roles_role_id_env ON users_roles(role_id, env); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_users_roles_role_id_env; +CREATE INDEX IF NOT EXISTS idx_users_roles_role_id ON users_roles(role_id); + +ALTER TABLE IF EXISTS users_roles DROP CONSTRAINT IF EXISTS users_roles_pkey; +ALTER TABLE IF EXISTS users_roles ADD PRIMARY KEY (user_id, role_id); +ALTER TABLE IF EXISTS users_roles DROP COLUMN IF EXISTS env; + +ALTER TABLE IF EXISTS async_searches DROP CONSTRAINT IF EXISTS async_searches_pkey; +ALTER TABLE IF EXISTS async_searches ADD PRIMARY KEY (search_id); +ALTER TABLE IF EXISTS async_searches DROP COLUMN IF EXISTS env; +-- +goose StatementEnd diff --git a/swagger/swagger.json b/swagger/swagger.json index 44b0a89..a812ae0 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -23,6 +23,14 @@ "admin_v1" ], "operationId": "admin_v1_get_roles", + "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + } + ], "responses": { "200": { "description": "A successful response", @@ -49,6 +57,12 @@ ], "operationId": "admin_v1_create_role", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -94,6 +108,12 @@ "name": "id", "in": "path", "required": true + }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" } ], "responses": { @@ -130,6 +150,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -170,6 +196,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -213,6 +245,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -254,6 +292,12 @@ "in": "path", "required": true }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -548,6 +592,12 @@ ], "operationId": "errorgroups_v1_get_details", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -586,6 +636,12 @@ ], "operationId": "errorgroups_v1_diff_by_releases", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -624,6 +680,12 @@ ], "operationId": "errorgroups_v1_get_groups", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -662,6 +724,12 @@ ], "operationId": "errorgroups_v1_get_hist", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -700,6 +768,12 @@ ], "operationId": "errorgroups_v1_get_releases", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -738,6 +812,12 @@ ], "operationId": "errorgroups_v1_get_service_filters", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -776,6 +856,12 @@ ], "operationId": "errorgroups_v1_get_services", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -814,6 +900,12 @@ ], "operationId": "errorgroups_v1_get_top_groups", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -1169,6 +1261,12 @@ ], "operationId": "seqapi_v1_fetch_async_search_result", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -1207,6 +1305,12 @@ ], "operationId": "seqapi_v1_get_async_searches_list", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -1245,6 +1349,12 @@ ], "operationId": "seqapi_v1_start_async_search", "parameters": [ + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" + }, { "description": "Request body", "name": "body", @@ -1289,6 +1399,12 @@ "name": "id", "in": "path", "required": true + }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" } ], "responses": { @@ -1322,6 +1438,12 @@ "name": "id", "in": "path", "required": true + }, + { + "type": "string", + "description": "Environment", + "name": "env", + "in": "query" } ], "responses": { @@ -3500,24 +3622,19 @@ "type": "object", "properties": { "maxAggregationsPerRequest": { - "type": "integer", - "format": "int32" + "type": "integer" }, "maxExportLimit": { - "type": "integer", - "format": "int32" + "type": "integer" }, "maxParallelExportRequests": { - "type": "integer", - "format": "int32" + "type": "integer" }, "maxSearchLimit": { - "type": "integer", - "format": "int32" + "type": "integer" }, "seqCliMaxSearchLimit": { - "type": "integer", - "format": "int32" + "type": "integer" } } }, @@ -3589,12 +3706,10 @@ } }, "limit": { - "type": "integer", - "format": "int32" + "type": "integer" }, "offset": { - "type": "integer", - "format": "int32" + "type": "integer" }, "offset_id": { "type": "string" diff --git a/tracing/tracing.go b/tracing/tracing.go index 43ccf2c..b73cb6a 100644 --- a/tracing/tracing.go +++ b/tracing/tracing.go @@ -14,7 +14,10 @@ import ( tracesdk "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.20.0" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/ozontech/seq-ui/logger" ) const ( @@ -28,19 +31,19 @@ type Config struct { SamplerParam float64 `env:"TRACING_SAMPLER_PARAM"` } -func Initialize() (*Config, error) { +func Init() error { tracingCfg, err := readConfig() if err != nil { - return nil, fmt.Errorf("failed to read tracing config: %w", err) + return fmt.Errorf("failed to read tracing config: %w", err) } if err := validateTracingConfig(tracingCfg); err != nil { - return nil, err + return err } tp, err := newTracerProvider(tracingCfg) if err != nil { - return nil, fmt.Errorf("can't create tracer provider: %w", err) + return fmt.Errorf("can't create tracer provider: %w", err) } // Register our TracerProvider as the global so any imported @@ -48,7 +51,14 @@ func Initialize() (*Config, error) { otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.TraceContext{}) - return tracingCfg, nil + logger.Info("tracing initialization success", + zap.String("service_name", tracingCfg.ServiceName), + zap.String("agent_host", tracingCfg.AgentHost), + zap.String("agent_port", tracingCfg.AgentPort), + zap.Float64("sampler_param", tracingCfg.SamplerParam), + ) + + return nil } func StartSpan(ctx context.Context, name string) (context.Context, trace.Span) {