From 23965c6356868031a50f4defd520edac3db5515d Mon Sep 17 00:00:00 2001 From: ttt161 Date: Tue, 12 May 2026 10:20:55 +0300 Subject: [PATCH] remove machinery --- .../generator/1770998661693-copy-bindings.sql | 0 .../sequence/1770994589112-copy-sequence.erl | 0 apps/bender/src/bender.app.src | 5 +- apps/bender/src/bender.erl | 37 +---- apps/bender/src/bender_generator.erl | 147 +----------------- apps/bender/src/bender_sequence.erl | 145 +---------------- apps/bender/src/bender_utils.erl | 54 ------- apps/bender/test/bender_ct_helper.erl | 72 +-------- apps/bender/test/bender_tests_SUITE.erl | 49 +----- apps/bender/test/generator_tests_SUITE.erl | 8 +- config/sys.config | 77 +-------- rebar.config | 2 +- rebar.lock | 30 +--- 13 files changed, 29 insertions(+), 597 deletions(-) rename apps/bender/priv/migrations/{ => from-progressor-deprecated}/generator/1770998661693-copy-bindings.sql (100%) rename apps/bender/priv/migrations/{ => from-progressor-deprecated}/sequence/1770994589112-copy-sequence.erl (100%) diff --git a/apps/bender/priv/migrations/generator/1770998661693-copy-bindings.sql b/apps/bender/priv/migrations/from-progressor-deprecated/generator/1770998661693-copy-bindings.sql similarity index 100% rename from apps/bender/priv/migrations/generator/1770998661693-copy-bindings.sql rename to apps/bender/priv/migrations/from-progressor-deprecated/generator/1770998661693-copy-bindings.sql diff --git a/apps/bender/priv/migrations/sequence/1770994589112-copy-sequence.erl b/apps/bender/priv/migrations/from-progressor-deprecated/sequence/1770994589112-copy-sequence.erl similarity index 100% rename from apps/bender/priv/migrations/sequence/1770994589112-copy-sequence.erl rename to apps/bender/priv/migrations/from-progressor-deprecated/sequence/1770994589112-copy-sequence.erl diff --git a/apps/bender/src/bender.app.src b/apps/bender/src/bender.app.src index e1ce6c2..5ae3c8c 100644 --- a/apps/bender/src/bender.app.src +++ b/apps/bender/src/bender.app.src @@ -11,9 +11,8 @@ prometheus_cowboy, woody, % woody's hackney declares prometheus metrics on start scoper, % should be before any scoper event handler usage - progressor, - epg_migrator, %% should be after epg_connector (progressor) - machinery, + epg_connector, + epg_migrator, %% should be after epg_connector erl_health, opentelemetry_api, opentelemetry_exporter, diff --git a/apps/bender/src/bender.erl b/apps/bender/src/bender.erl index 6de84dc..528600f 100644 --- a/apps/bender/src/bender.erl +++ b/apps/bender/src/bender.erl @@ -38,7 +38,7 @@ stop() -> -spec start(normal, any()) -> {ok, pid()} | {error, any()}. start(_StartType, _StartArgs) -> ok = setup_metrics(), - ok = db_init(application:get_env(bender, backend_mode, machinery)), + ok = db_init(), supervisor:start_link({local, ?MODULE}, ?MODULE, []). -spec stop(any()) -> ok. @@ -60,7 +60,7 @@ init([]) -> shutdown_timeout => get_shutdown_timeout(), event_handler => EventHandlers, handlers => get_handler_spec(), - additional_routes => get_routes(EventHandlers, genlib_app:env(?MODULE, machinery_backend)) + additional_routes => get_routes(EventHandlers) } ), Flags = #{strategy => one_for_all, intensity => 6, period => 30}, @@ -105,32 +105,11 @@ get_handler_spec() -> }} ]. --spec get_routes(woody:ev_handlers(), machinegun | progressor | hybrid) -> [woody_server_thrift_http_handler:route(_)]. -get_routes(_EventHandlers, progressor) -> +-spec get_routes(woody:ev_handlers()) -> [woody_server_thrift_http_handler:route(_)]. +get_routes(_EventHandlers) -> %% Shared routes Check = enable_health_logging(genlib_app:env(?MODULE, health_check, #{})), - [erl_health_handle:get_route(Check), get_prometheus_route()]; -get_routes(EventHandlers, Mode) when Mode == machinegun orelse Mode == hybrid -> - %% Machinegun specific routes - RouteOptsEnv = genlib_app:env(?MODULE, route_opts, #{}), - RouteOpts = RouteOptsEnv#{event_handler => EventHandlers}, - Generator = genlib_app:env(bender, generator, #{}), - Sequence = genlib_app:env(bender, sequence, #{}), - Handlers = [ - {bender_generator, #{ - path => maps:get(path, Generator, <<"/v1/stateproc/bender_generator">>), - backend_config => #{ - schema => maps:get(schema, Generator, machinery_mg_schema_generic) - } - }}, - {bender_sequence, #{ - path => maps:get(path, Sequence, <<"/v1/stateproc/bender_sequence">>), - backend_config => #{ - schema => maps:get(schema, Sequence, machinery_mg_schema_generic) - } - }} - ], - get_routes(EventHandlers, progressor) ++ machinery_mg_backend:get_routes(Handlers, RouteOpts). + [erl_health_handle:get_route(Check), get_prometheus_route()]. -spec enable_health_logging(erl_health:check()) -> erl_health:check(). enable_health_logging(Check) -> @@ -148,7 +127,7 @@ setup_metrics() -> ok = woody_ranch_prometheus_collector:setup(), ok = woody_hackney_prometheus_collector:setup(). -db_init(postgres) -> +db_init() -> case code:priv_dir(bender) of {error, _} -> error({migration_error, cant_find_priv_dir}); @@ -159,9 +138,7 @@ db_init(postgres) -> {ok, Databases} = application:get_env(epg_connector, databases), DbOpts = maps:get(DbRef, Databases), db_init(MigrationsDir, DbOpts) - end; -db_init(machinery) -> - ok. + end. db_init(MigrationsDir, DbOpts) -> MigrationOpts = application:get_env(bender, migration_opts, ?DEFAULT_MIGRATION_OPTS), diff --git a/apps/bender/src/bender_generator.erl b/apps/bender/src/bender_generator.erl index 4536318..8db7b86 100644 --- a/apps/bender/src/bender_generator.erl +++ b/apps/bender/src/bender_generator.erl @@ -6,60 +6,20 @@ -export([generate/2]). -export([get_internal_id/2]). -%% Machinery callbacks - --behaviour(machinery). - --export([init/4]). --export([process_call/4]). --export([process_timeout/3]). --export([process_repair/4]). --export([process_notification/4]). - -type external_id() :: binary(). -type internal_id() :: binary() | {binary(), pos_integer()}. -type schema() :: bender:schema(). -type user_context() :: msgp_msgpack_thrift:'Value'() | undefined. --type state() :: #{ - internal_id := internal_id(), - user_context := user_context() -}. - -type woody_context() :: woody_context:ctx(). --type args(T) :: machinery:args(T). --type machine() :: machinery:machine(_, state()). --type handler_args() :: machinery:handler_args(_). --type handler_opts() :: machinery:handler_opts(_). --type result(A) :: machinery:result(none(), A). - -include("bender_internal.hrl"). --define(NS, bender_generator). - %%% API -spec bind(external_id(), schema(), user_context(), woody_context()) -> {ok, internal_id(), user_context()} | no_return(). bind(ExternalID, Schema, UserCtx, WoodyCtx) -> - case backend_mode() of - machinery -> - bind_via_machinery(ExternalID, Schema, UserCtx, WoodyCtx); - postgres -> - bind_via_postgres(ExternalID, Schema, UserCtx, WoodyCtx) - end. - -bind_via_machinery(ExternalID, Schema, UserCtx, WoodyCtx) -> - InternalID = generate(Schema, WoodyCtx), - case start(ExternalID, InternalID, UserCtx, WoodyCtx) of - ok -> - {ok, InternalID, undefined}; - {error, exists} -> - get_internal_id_with_retry(ExternalID, WoodyCtx) - end. - -bind_via_postgres(ExternalID, Schema, UserCtx, WoodyCtx) -> InternalID = generate(Schema, WoodyCtx), SQL = "INSERT INTO bender_generator_states (id, state) values ($1, $2) ON CONFLICT (id) DO NOTHING RETURNING state", State = term_to_binary( @@ -73,7 +33,7 @@ bind_via_postgres(ExternalID, Schema, UserCtx, WoodyCtx) -> case Result of {ok, _, _, []} -> %% already inserted - get_internal_id_via_postgres(ExternalID, WoodyCtx); + get_internal_id(ExternalID, WoodyCtx); {ok, _, _, [{_SavedState}]} -> %% first insert {ok, InternalID, undefined}; @@ -83,27 +43,7 @@ bind_via_postgres(ExternalID, Schema, UserCtx, WoodyCtx) -> end. -spec get_internal_id(external_id(), woody_context()) -> {ok, internal_id(), user_context()} | no_return(). -get_internal_id(ExternalID, WoodyCtx) -> - case backend_mode() of - machinery -> - get_internal_id_via_machinery(ExternalID, WoodyCtx); - postgres -> - get_internal_id_via_postgres(ExternalID, WoodyCtx) - end. - -get_internal_id_via_machinery(ExternalID, WoodyCtx) -> - case machinery:get(?NS, ExternalID, get_backend(WoodyCtx)) of - {ok, Machine} -> - #{ - internal_id := InternalID, - user_context := UserCtx - } = get_machine_state(Machine), - {ok, InternalID, UserCtx}; - {error, notfound} -> - throw({not_found, ExternalID}) - end. - -get_internal_id_via_postgres(ExternalID, _WoodyCtx) -> +get_internal_id(ExternalID, _WoodyCtx) -> Pool = application:get_env(bender, generator_pool, default_pool), SQL = "SELECT state FROM bender_generator_states WHERE id = $1", Result = epg_pool:query(Pool, SQL, [ExternalID]), @@ -121,86 +61,6 @@ get_internal_id_via_postgres(ExternalID, _WoodyCtx) -> error({internal_error, Error}) end. --spec get_internal_id_with_retry(external_id(), woody_context()) -> {ok, internal_id(), user_context()} | no_return(). -get_internal_id_with_retry(ExternalID, WoodyCtx) -> - with_retry(get_retry_strategy(), fun() -> - try - {done, get_internal_id(ExternalID, WoodyCtx)} - catch - %% NOTE Underlying machinery backend can experience race - %% condition on machine writes. Thus it MAY occur that - %% 'aux_state' is undefined on machine read. - error:({badmatch, undefined}):_Stacktrace -> - retry - end - end). - -%%% Machinery callbacks - --spec init(args({internal_id(), user_context()}), machine(), handler_args(), handler_opts()) -> result(state()). -init({InternalID, UserCtx}, _Machine, _HandlerArgs, _HandlerOpts) -> - #{ - aux_state => #{ - internal_id => InternalID, - user_context => UserCtx - } - }. - --spec process_call(args(_), machine(), handler_args(), handler_opts()) -> no_return(). -process_call(_Args, _Machine, _HandlerArgs, _HandlerOpts) -> - not_implemented(call). - --spec process_timeout(machine(), handler_args(), handler_opts()) -> no_return(). -process_timeout(_Machine, _HandlerArgs, _HandlerOpts) -> - not_implemented(timeout). - --spec process_repair(args(_), machine(), handler_args(), handler_opts()) -> no_return(). -process_repair(_Args, _Machine, _HandlerArgs, _HandlerOpts) -> - not_implemented(repair). - --spec process_notification(args(_), machine(), handler_args(), handler_opts()) -> no_return(). -process_notification(_Args, _Machine, _HandlerArgs, _HandlerOpts) -> - not_implemented(notification). - -%%% Internal functions - --spec get_retry_strategy() -> genlib_retry:strategy(). -get_retry_strategy() -> - Opts = genlib_app:env(bender, generator), - DefaultPolicy = genlib_retry:exponential(5, 2, {jitter, 200, 100}), - genlib_retry:new_strategy(maps:get(retry_policy, Opts, DefaultPolicy)). - --spec with_retry(genlib_retry:strategy(), fun(() -> {done, T} | retry)) -> T | no_return(). -with_retry(Strategy, Fun) -> - case Fun() of - {done, Result} -> - Result; - retry -> - case genlib_retry:next_step(Strategy) of - {wait, Timeout, NextStrategy} -> - _ = timer:sleep(Timeout), - with_retry(NextStrategy, Fun); - finish -> - erlang:error(retries_exhausted) - end - end. - --spec start(external_id(), internal_id(), user_context(), woody_context()) -> ok | {error, exists}. -start(ExternalID, InternalID, UserCtx, WoodyCtx) -> - machinery:start(?NS, ExternalID, {InternalID, UserCtx}, get_backend(WoodyCtx)). - --spec get_machine_state(machine()) -> state(). -get_machine_state(#{aux_state := State}) -> - State. - --spec get_backend(woody_context()) -> machinery_mg_backend:backend(). -get_backend(WoodyCtx) -> - bender_utils:get_backend(generator, WoodyCtx). - --spec not_implemented(any()) -> no_return(). -not_implemented(What) -> - erlang:error({not_implemented, What}). - -spec generate(schema(), woody_context()) -> internal_id(). generate(snowflake, _WoodyCtx) -> <> = snowflake:new(), @@ -211,6 +71,3 @@ generate(#constant{internal_id = InternalID}, _WoodyCtx) -> generate(#sequence{id = SequenceID, minimum = Minimum}, WoodyCtx) -> {ok, IntegerID} = bender_sequence:get_next(SequenceID, Minimum, WoodyCtx), {integer_to_binary(IntegerID), IntegerID}. - -backend_mode() -> - application:get_env(bender, backend_mode, machinery). diff --git a/apps/bender/src/bender_sequence.erl b/apps/bender/src/bender_sequence.erl index 851791b..86ede04 100644 --- a/apps/bender/src/bender_sequence.erl +++ b/apps/bender/src/bender_sequence.erl @@ -6,16 +6,6 @@ -export([get_next/2]). -export([get_next/3]). -%% Machinery callbacks - --behaviour(machinery). - --export([init/4]). --export([process_call/4]). --export([process_timeout/3]). --export([process_repair/4]). --export([process_notification/4]). - -type id() :: binary(). -type minimum() :: integer() | undefined. @@ -24,46 +14,14 @@ -type woody_context() :: woody_context:ctx(). --type args(T) :: machinery:args(T). --type machine() :: machinery:machine(_, state()). --type handler_args() :: machinery:handler_args(_). --type handler_opts() :: machinery:handler_opts(_). --type result(A) :: machinery:result(none(), A). --type response(T) :: machinery:response(T). - -type value() :: non_neg_integer(). --type state() :: #{ - value := value() -}. - --type init_args() :: #{ - initial_value => value() -}. - --define(NS, bender_sequence). -define(DEFAULT_INITIAL_VALUE, 1). %%% API -spec get_current(id(), woody_context()) -> {ok, value()} | {error, notfound}. -get_current(SequenceID, WoodyCtx) -> - case backend_mode() of - machinery -> - get_current_via_machinery(SequenceID, WoodyCtx); - postgres -> - get_current_via_postgres(SequenceID, WoodyCtx) - end. - -get_current_via_machinery(SequenceID, WoodyCtx) -> - case get_state(SequenceID, WoodyCtx) of - {ok, State} -> - {ok, get_value(State)}; - _ -> - {error, notfound} - end. - -get_current_via_postgres(SequenceID, _WoodyCtx) -> +get_current(SequenceID, _WoodyCtx) -> SQL = "SELECT value FROM bender_sequence_values WHERE id = $1", Result = epg_pool:query(pg_pool(), SQL, [SequenceID]), case Result of @@ -83,24 +41,7 @@ get_next(SequenceID, WoodyCtx) -> -spec get_next(id(), minimum(), woody_context()) -> {ok, value()}. get_next(SequenceID, undefined, WoodyCtx) -> get_next(SequenceID, ?DEFAULT_INITIAL_VALUE, WoodyCtx); -get_next(SequenceID, Minimum, WoodyCtx) -> - case backend_mode() of - machinery -> - get_next_via_machinery(SequenceID, Minimum, WoodyCtx); - postgres -> - get_next_via_postgres(SequenceID, Minimum, WoodyCtx) - end. - -get_next_via_machinery(SequenceID, Minimum, WoodyCtx) -> - Args = #{initial_value => Minimum}, - case start(SequenceID, Args, WoodyCtx) of - ok -> - {ok, Minimum}; - {error, exists} -> - call(SequenceID, {get_next, Minimum}, WoodyCtx) - end. - -get_next_via_postgres(SequenceID, Minimum, _WoodyCtx) -> +get_next(SequenceID, Minimum, _WoodyCtx) -> SQL = "INSERT INTO bender_sequence_values (id, value) values ($1, $2) " " ON CONFLICT (id) DO UPDATE SET value = GREATEST(bender_sequence_values.value + 1, $2) RETURNING value", @@ -113,87 +54,5 @@ get_next_via_postgres(SequenceID, Minimum, _WoodyCtx) -> error({sequence_increment_error, Error}) end. -%%% Machinery callbacks - --spec init(args(init_args()), machine(), handler_args(), handler_opts()) -> result(state()). -init(Args, _Machine, _HandlerArgs, _HandlerOpts) -> - #{ - aux_state => #{ - value => maps:get(initial_value, Args, ?DEFAULT_INITIAL_VALUE) - } - }. - --spec process_call(args({get_next, integer()} | any()), machine(), handler_args(), handler_opts()) -> - {response(value()), result(state())} | no_return(). -process_call({get_next, Minimum}, Machine, _HandlerArgs, _HandlerOpts) -> - State = get_machine_state(Machine), - Value = get_value(State), - NewValue = erlang:max(Value + 1, Minimum), - NewState = set_value(State, NewValue), - {NewValue, #{aux_state => NewState}}; -process_call(_Args, _Machine, _HandlerArgs, _HandlerOpts) -> - not_implemented(call). - --spec process_timeout(machine(), handler_args(), handler_opts()) -> no_return(). -process_timeout(_Machine, _HandlerArgs, _HandlerOpts) -> - not_implemented(timeout). - --spec process_repair(args(_), machine(), handler_args(), handler_opts()) -> no_return(). -process_repair(_Args, _Machine, _HandlerArgs, _HandlerOpts) -> - not_implemented(repair). - --spec process_notification(args(_), machine(), handler_args(), handler_opts()) -> no_return(). -process_notification(_Args, _Machine, _HandlerArgs, _HandlerOpts) -> - not_implemented(notification). - -%%% Internal functions - --spec start(id(), init_args(), woody_context()) -> ok | {error, exists}. -start(SequenceID, Args, WoodyCtx) -> - machinery:start(?NS, SequenceID, Args, get_backend(WoodyCtx)). - --spec call(id(), args(_), woody_context()) -> {ok, response(_)} | {error, notfound}. -call(SequenceID, Msg, WoodyCtx) -> - case machinery:call(?NS, SequenceID, Msg, get_backend(WoodyCtx)) of - {error, notfound} -> - timer:sleep(100), - call(SequenceID, Msg, WoodyCtx); - Result -> - Result - end. - --spec get_state(id(), woody_context()) -> {ok, state()} | {error, notfound}. -get_state(SequenceID, WoodyCtx) -> - case machinery:get(?NS, SequenceID, get_backend(WoodyCtx)) of - {ok, Machine} -> - State = get_machine_state(Machine), - {ok, State}; - {error, notfound} = Error -> - Error - end. - --spec get_machine_state(machine()) -> state(). -get_machine_state(#{aux_state := State}) -> - State. - --spec get_backend(woody_context()) -> machinery_mg_backend:backend(). -get_backend(WoodyCtx) -> - bender_utils:get_backend(sequence, WoodyCtx). - --spec not_implemented(any()) -> no_return(). -not_implemented(What) -> - erlang:error({not_implemented, What}). - --spec get_value(state()) -> value(). -get_value(#{value := Value}) -> - Value. - --spec set_value(state(), value()) -> state(). -set_value(State, Value) -> - State#{value => Value}. - -backend_mode() -> - application:get_env(bender, backend_mode, machinery). - pg_pool() -> application:get_env(bender, sequence_pool, default_pool). diff --git a/apps/bender/src/bender_utils.erl b/apps/bender/src/bender_utils.erl index d205b6e..742e973 100644 --- a/apps/bender/src/bender_utils.erl +++ b/apps/bender/src/bender_utils.erl @@ -1,24 +1,8 @@ -module(bender_utils). -export([unique_id/0]). --export([get_backend/2]). -export([get_woody_event_handlers/0]). --type woody_context() :: woody_context:ctx(). - --type schema() :: machinery_mg_schema_generic | atom(). --type event_handler() :: woody:ev_handler() | [woody:ev_handler()]. - --type automaton() :: #{ - % machinegun's automaton url - url := binary(), - event_handler := event_handler(), - % state processor path - path => binary(), - schema => schema(), - transport_opts => woody_client_thrift_http_transport:transport_options() -}. - %%% API -spec unique_id() -> binary(). @@ -26,46 +10,8 @@ unique_id() -> <> = snowflake:new(), genlib_format:format_int_base(ID, 62). --spec get_backend(generator | sequence, woody_context()) -> machinery_mg_backend:backend(). -get_backend(Service, WoodyCtx) -> - get_backend(genlib_app:env(bender, machinery_backend), Service, WoodyCtx). - %%% Internal functions -get_backend(hybrid, Service, WoodyCtx) -> - {machinery_hybrid_backend, #{ - primary_backend => get_backend(progressor, Service, WoodyCtx), - fallback_backend => get_backend(machinegun, Service, WoodyCtx) - }}; -get_backend(progressor, Service, WoodyCtx) -> - Automaton = genlib_app:env(bender, Service, #{}), - {Namespace, Handler} = get_machinery_namespace_handler(Service), - machinery_prg_backend:new(WoodyCtx, #{ - namespace => Namespace, - handler => Handler, - schema => maps:get(schema, Automaton, machinery_mg_schema_generic) - }); -get_backend(machinegun, Service, WoodyCtx) -> - Automaton = genlib_app:env(bender, Service, #{}), - machinery_mg_backend:new(WoodyCtx, #{ - client => get_woody_client(Automaton), - schema => maps:get(schema, Automaton, machinery_mg_schema_generic) - }). - --spec get_machinery_namespace_handler(generator | sequence) -> {machinery:namespace(), machinery:logic_handler(_)}. -get_machinery_namespace_handler(generator) -> - {'bender_generator', {bender_generator, #{}}}; -get_machinery_namespace_handler(sequence) -> - {'bender_sequence', {bender_sequence, #{}}}. - --spec get_woody_client(automaton()) -> machinery_mg_client:woody_client(). -get_woody_client(#{url := Url} = Automaton) -> - genlib_map:compact(#{ - url => Url, - event_handler => get_woody_event_handlers(), - transport_opts => maps:get(transport_opts, Automaton, undefined) - }). - -spec get_woody_event_handlers() -> woody:ev_handlers(). get_woody_event_handlers() -> genlib_app:env(bender, woody_event_handlers, [ diff --git a/apps/bender/test/bender_ct_helper.erl b/apps/bender/test/bender_ct_helper.erl index fcbc01c..dd19e08 100644 --- a/apps/bender/test/bender_ct_helper.erl +++ b/apps/bender/test/bender_ct_helper.erl @@ -1,11 +1,11 @@ -module(bender_ct_helper). --export([start_apps/2]). +-export([start_apps/1]). -type config() :: [{atom(), term()}]. --spec start_apps(machinery | postgres, config()) -> config(). -start_apps(BackendMode, C) -> +-spec start_apps(config()) -> config(). +start_apps(C) -> EpgConnectorApps = genlib_app:start_application_with(epg_connector, [ {databases, #{ default_db => #{ @@ -23,75 +23,11 @@ start_apps(BackendMode, C) -> } }} ]), - ProgressorApps = genlib_app:start_application_with(progressor, [ - {call_wait_timeout, 20}, - {defaults, #{ - storage => #{ - client => prg_pg_backend, - options => #{ - pool => default_pool - } - }, - retry_policy => #{ - initial_timeout => 5, - backoff_coefficient => 1.0, - %% seconds - max_timeout => 180, - max_attempts => 3, - non_retryable_errors => [] - }, - task_scan_timeout => 1, - worker_pool_size => 1000, - process_step_timeout => 30 - }}, - {namespaces, #{ - 'bender_generator' => #{ - processor => #{ - client => machinery_prg_backend, - options => #{ - namespace => 'bender_generator', - handler => {bender_generator, #{}}, - schema => machinery_mg_schema_generic - } - } - }, - 'bender_sequence' => #{ - processor => #{ - client => machinery_prg_backend, - options => #{ - namespace => 'bender_sequence', - handler => {bender_sequence, #{}}, - schema => machinery_mg_schema_generic - } - } - } - }} - ]), ScoperApps = genlib_app:start_application_with(scoper, [ {storage, scoper_storage_logger} ]), BenderApps = genlib_app:start_application_with(bender, [ {db_ref, default_db}, - {backend_mode, BackendMode}, - {machinery_backend, hybrid}, - {generator, #{ - path => <<"/v1/stateproc/bender_generator">>, - schema => machinery_mg_schema_generic, - url => <<"http://machinegun:8022/v1/automaton">>, - event_handler => scoper_woody_event_handler, - transport_opts => #{ - max_connections => 1000 - } - }}, - {sequence, #{ - path => <<"/v1/stateproc/bender_sequence">>, - schema => machinery_mg_schema_generic, - url => <<"http://machinegun:8022/v1/automaton">>, - event_handler => scoper_woody_event_handler, - transport_opts => #{ - max_connections => 1000 - } - }}, {protocol_opts, #{ timeout => 60000 }}, @@ -100,5 +36,5 @@ start_apps(BackendMode, C) -> num_acceptors => 100 }} ]), - Apps = EpgConnectorApps ++ ProgressorApps ++ ScoperApps ++ BenderApps, + Apps = EpgConnectorApps ++ ScoperApps ++ BenderApps, [{suite_apps, Apps} | C]. diff --git a/apps/bender/test/bender_tests_SUITE.erl b/apps/bender/test/bender_tests_SUITE.erl index 8bcc846..7c8cc44 100644 --- a/apps/bender/test/bender_tests_SUITE.erl +++ b/apps/bender/test/bender_tests_SUITE.erl @@ -17,7 +17,6 @@ -export([different_schemas/1]). -export([contention/1]). --export([generator_init/1]). -export([retrieve_unknown_id/1]). -export([retrieve_known_id/1]). @@ -32,7 +31,6 @@ -spec all() -> [test_case_name() | {group, group_name()}]. all() -> [ - {group, machinery}, {group, postgres} ]. @@ -42,11 +40,6 @@ all() -> -spec groups() -> [{group_name(), list(), [test_case_name()]}]. groups() -> [ - {machinery, [], [ - {group, main}, - {group, contention}, - {group, retrieve_id} - ]}, {postgres, [], [ {group, main}, {group, contention}, @@ -57,14 +50,12 @@ groups() -> {group, sequence}, sequence_minimum, {group, snowflake}, - {group, different_schemas}, - {group, generator_init} + {group, different_schemas} ]}, {constant, [parallel], [constant || _ <- lists:seq(1, ?PARALLEL_WORKERS)]}, {sequence, [parallel], [sequence || _ <- lists:seq(1, ?PARALLEL_WORKERS)]}, {snowflake, [parallel], [snowflake || _ <- lists:seq(1, ?PARALLEL_WORKERS)]}, {different_schemas, [parallel], [different_schemas || _ <- lists:seq(1, ?PARALLEL_WORKERS)]}, - {generator_init, [parallel], [generator_init || _ <- lists:seq(1, ?PARALLEL_WORKERS)]}, {contention, [{repeat_until_all_ok, 10}], [ contention ]}, @@ -83,13 +74,13 @@ end_per_suite(_C) -> ok. -spec init_per_group(atom(), config()) -> config(). -init_per_group(Group, C) when Group =:= machinery; Group =:= postgres -> - bender_ct_helper:start_apps(Group, C); +init_per_group(Group, C) when Group =:= postgres -> + bender_ct_helper:start_apps(C); init_per_group(_Group, C) -> C. -spec end_per_group(atom(), config()) -> ok. -end_per_group(Group, C) when Group =:= machinery; Group =:= postgres -> +end_per_group(Group, C) when Group =:= postgres -> genlib_app:stop_unload_applications(?CONFIG(suite_apps, C)); end_per_group(_Group, _C) -> ok. @@ -199,38 +190,6 @@ contention(C) -> UserCtxOfWinner = binary_to_term(BinaryCtx), ok. --include_lib("mg_proto/include/mg_proto_state_processing_thrift.hrl"). - --spec generator_init(config()) -> ok. -generator_init(_C) -> - Request = #mg_stateproc_SignalArgs{ - signal = { - init, - #mg_stateproc_InitSignal{ - arg = {arr, [{str, <<"tup">>}, {str, <<"snowflake">>}, {bin, <<"user context">>}]} - } - }, - machine = #mg_stateproc_Machine{ - ns = <<"bender_generator">>, - id = <<"42">>, - history = [], - history_range = #mg_stateproc_HistoryRange{}, - aux_state = #mg_stateproc_Content{data = {bin, <<>>}}, - timer = undefined - } - }, - Call = {{mg_proto_state_processing_thrift, 'Processor'}, 'ProcessSignal', {Request}}, - Options = #{ - url => <<"http://localhost:8022/v1/stateproc/bender_generator">>, - event_handler => scoper_woody_event_handler, - transport_opts => #{ - checkout_timeout => 1000, - max_connections => 10000 - } - }, - {ok, _Result} = woody_client:call(Call, Options), - ok. - -spec retrieve_unknown_id(config()) -> ok. retrieve_unknown_id(C) -> Client = get_client(C), diff --git a/apps/bender/test/generator_tests_SUITE.erl b/apps/bender/test/generator_tests_SUITE.erl index 4b1d44e..f7f69d1 100644 --- a/apps/bender/test/generator_tests_SUITE.erl +++ b/apps/bender/test/generator_tests_SUITE.erl @@ -25,7 +25,6 @@ -spec all() -> [test_case_name() | {group, group_name()}]. all() -> [ - {group, machinery}, {group, postgres} ]. @@ -34,7 +33,6 @@ all() -> -spec groups() -> [{group_name(), list(), [test_case_name()]}]. groups() -> [ - {machinery, [], [{group, main}]}, {postgres, [], [{group, main}]}, {main, [parallel], [ sequence, @@ -53,13 +51,13 @@ end_per_suite(_C) -> ok. -spec init_per_group(atom(), config()) -> config(). -init_per_group(Group, C) when Group =:= machinery; Group =:= postgres -> - bender_ct_helper:start_apps(Group, C); +init_per_group(Group, C) when Group =:= postgres -> + bender_ct_helper:start_apps(C); init_per_group(_Group, C) -> C. -spec end_per_group(atom(), config()) -> ok. -end_per_group(Group, C) when Group =:= machinery; Group =:= postgres -> +end_per_group(Group, C) when Group =:= postgres -> genlib_app:stop_unload_applications(?CONFIG(suite_apps, C)); end_per_group(_Group, _C) -> ok. diff --git a/config/sys.config b/config/sys.config index 2162ff7..1e61f47 100644 --- a/config/sys.config +++ b/config/sys.config @@ -1,51 +1,21 @@ [ {bender, [ - %% Available options for 'backend_mode' - %% machinery (by default) | postgres - %% - %% For 'postgres' mode optional parameters are supported: + %% Optional parameters are supported: %% 'db_ref' ::atom() ('bender' by default) %% used to perform migrations only %% must be key exists from epg_connector.databases map %% 'sequence_pool' :: atom() ('default_pool' by default) %% 'generator_pool' :: atom() ('default_pool' by default) %% must be key exists from epg_connector.pools map - {backend_mode, postgres}, {db_ref, default_db}, {sequence_pool, default_pool}, {generator_pool, default_pool}, - %% Available options for 'machinery_backend' - %% machinegun | progressor | hybrid - %% - %% For 'progressor' and 'hybrid' backends ensure config - %% '{progressor, [ ... ]}' is set. - {machinery_backend, hybrid}, {services, #{ bender => #{path => <<"/v1/bender">>}, generator => #{path => <<"/v1/generator">>} }}, - {generator, #{ - path => <<"/v1/stateproc/bender_generator">>, - schema => machinery_mg_schema_generic, - % mandatory - url => <<"http://machinegun:8022/v1/automaton">>, - transport_opts => #{ - max_connections => 1000 - } - }}, - - {sequence, #{ - path => <<"/v1/stateproc/bender_sequence">>, - schema => machinery_mg_schema_generic, - % mandatory - url => <<"http://machinegun:8022/v1/automaton">>, - transport_opts => #{ - max_connections => 1000 - } - }}, - {route_opts, #{ % handler_limits => #{} }}, @@ -128,51 +98,6 @@ }} ]}, - {progressor, [ - {call_wait_timeout, 20}, - {defaults, #{ - storage => #{ - client => prg_pg_backend, - options => #{ - pool => default_pool - } - }, - retry_policy => #{ - initial_timeout => 5, - backoff_coefficient => 1.0, - %% seconds - max_timeout => 180, - max_attempts => 3, - non_retryable_errors => [] - }, - task_scan_timeout => 1, - worker_pool_size => 100, - process_step_timeout => 30 - }}, - {namespaces, #{ - 'bender_generator' => #{ - processor => #{ - client => machinery_prg_backend, - options => #{ - namespace => 'bender_generator', - handler => {bender_generator, #{}}, - schema => machinery_mg_schema_generic - } - } - }, - 'bender_sequence' => #{ - processor => #{ - client => machinery_prg_backend, - options => #{ - namespace => 'bender_sequence', - handler => {bender_sequence, #{}}, - schema => machinery_mg_schema_generic - } - } - } - }} - ]}, - {os_mon, [ % for better compatibility with busybox coreutils {disksup_posix_only, true} diff --git a/rebar.config b/rebar.config index a27d3c2..2e93b67 100644 --- a/rebar.config +++ b/rebar.config @@ -31,7 +31,7 @@ {bender_proto, {git, "https://github.com/valitydev/bender-proto.git", {branch, "master"}}}, {erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, "master"}}}, {genlib, {git, "https://github.com/valitydev/genlib.git", {tag, "v1.1.0"}}}, - {machinery, {git, "https://github.com/valitydev/machinery-erlang.git", {tag, "v1.1.19"}}}, + {epg_connector, {git, "https://github.com/valitydev/epg_connector.git", {tag, "v0.0.5"}}}, {epg_migrator, {git, "https://github.com/valitydev/epg_migrator.git", {tag, "v0.0.2"}}}, {scoper, {git, "https://github.com/valitydev/scoper.git", {tag, "v1.1.0"}}}, {snowflake, {git, "https://github.com/valitydev/snowflake.git", {branch, "master"}}}, diff --git a/rebar.lock b/rebar.lock index d5a76bd..c7b3088 100644 --- a/rebar.lock +++ b/rebar.lock @@ -5,12 +5,11 @@ {git,"https://github.com/valitydev/bender-proto.git", {ref,"753b935b52a52e41b571d6e580f7dfe1377364f1"}}, 0}, - {<<"brod">>,{pkg,<<"brod">>,<<"4.3.2">>},2}, {<<"cache">>,{pkg,<<"cache">>,<<"2.3.3">>},1}, {<<"canal">>, {git,"https://github.com/valitydev/canal", {ref,"89faedce3b054bcca7cc31ca64d2ead8a9402305"}}, - 3}, + 1}, {<<"certifi">>,{pkg,<<"certifi">>,<<"2.8.0">>},2}, {<<"cg_mon">>, {git,"https://github.com/rbkmoney/cg_mon.git", @@ -19,12 +18,11 @@ {<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.15.1">>},2}, {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},1}, {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.11.0">>},2}, - {<<"crc32cer">>,{pkg,<<"crc32cer">>,<<"0.1.11">>},4}, {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},2}, {<<"epg_connector">>, {git,"https://github.com/valitydev/epg_connector.git", {ref,"939a0d4ab3f7561a79b45381bbe13029d9263006"}}, - 2}, + 0}, {<<"epg_migrator">>, {git,"https://github.com/valitydev/epg_migrator.git", {ref,"8633c43fb4d022355c1498c0effa9ecf4098a67f"}}, @@ -44,18 +42,9 @@ {<<"hackney">>,{pkg,<<"hackney">>,<<"1.18.0">>},1}, {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.3.0">>},3}, {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},2}, - {<<"jsone">>,{pkg,<<"jsone">>,<<"1.8.0">>},4}, + {<<"jsone">>,{pkg,<<"jsone">>,<<"1.8.0">>},2}, {<<"jsx">>,{pkg,<<"jsx">>,<<"3.1.0">>},1}, - {<<"kafka_protocol">>,{pkg,<<"kafka_protocol">>,<<"4.1.10">>},3}, - {<<"machinery">>, - {git,"https://github.com/valitydev/machinery-erlang.git", - {ref,"eac35324e9adc7bfc52e7bb83148335cba01fee8"}}, - 0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},2}, - {<<"mg_proto">>, - {git,"https://github.com/valitydev/machinegun-proto", - {ref,"cc2c27c30d30dc34c0c56fc7c7e96326d6bd6a14"}}, - 1}, {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.4.0">>},2}, {<<"msgpack_proto">>, {git,"https://github.com/valitydev/msgpack-proto.git", @@ -70,16 +59,11 @@ {pkg,<<"opentelemetry_semantic_conventions">>,<<"0.2.0">>}, 1}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.1">>},2}, - {<<"progressor">>, - {git,"https://github.com/valitydev/progressor.git", - {ref,"6033631d3e1eb9593acf7841d8a635146ff482e8"}}, - 1}, {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.8.1">>},0}, {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.8">>},0}, {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.15">>},1}, {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},1}, {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},2}, - {<<"recon">>,{pkg,<<"recon">>,<<"2.5.6">>},2}, {<<"scoper">>, {git,"https://github.com/valitydev/scoper.git", {ref,"0e7aa01e9632daa39727edd62d4656ee715b4569"}}, @@ -105,13 +89,11 @@ {pkg_hash,[ {<<"accept">>, <<"CD6E34A2D7E28CA38B2D3CB233734CA0C221EFBC1F171F91FEC5F162CC2D18DA">>}, {<<"acceptor_pool">>, <<"43C20D2ACAE35F0C2BCD64F9D2BDE267E459F0F3FD23DAB26485BF518C281B21">>}, - {<<"brod">>, <<"51F4DFF17ED43A806558EBD62CC88E7B35AED336D1BA1F3DE2D010F463D49736">>}, {<<"cache">>, <<"B23A5FE7095445A88412A6E614C933377E0137B44FFED77C9B3FEF1A731A20B2">>}, {<<"certifi">>, <<"D4FB0A6BB20B7C9C3643E22507E42F356AC090A1DCEA9AB99E27E0376D695EBA">>}, {<<"chatterbox">>, <<"5CAC4D15DD7AD61FC3C4415CE4826FC563D4643DEE897A558EC4EA0B1C835C9C">>}, {<<"cowboy">>, <<"865DD8B6607E14CF03282E10E934023A1BD8BE6F6BACF921A7E2A96D800CD452">>}, {<<"cowlib">>, <<"0B9FF9C346629256C42EBE1EEB769A83C6CB771A6EE5960BD110AB0B9B872063">>}, - {<<"crc32cer">>, <<"B550DA6D615FEB72A882D15D020F8F7DEE72DFB2CB1BCDF3B1EE8DC2AFD68CFC">>}, {<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>}, {<<"epgsql">>, <<"D4E47CAE46C18C8AFA88E34D59A9B4BAE16368D7CE1EB3DA24FA755EB28393EB">>}, {<<"erlydtl">>, <<"964B2DC84F8C17ACFAA69C59BA129EF26AC45D2BA898C3C6AD9B5BDC8BA13CED">>}, @@ -122,7 +104,6 @@ {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, {<<"jsone">>, <<"347FF1FA700E182E1F9C5012FA6D737B12C854313B9AE6954CA75D3987D6C06D">>}, {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>}, - {<<"kafka_protocol">>, <<"F917B6C90C8DF0DE2B40A87D6B9AE1CFCE7788E91A65818E90E40CF76111097A">>}, {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, {<<"mimerl">>, <<"3882A5CA67FBBE7117BA8947F27643557ADEC38FA2307490C4C4207624CB213B">>}, {<<"opentelemetry">>, <<"988AC3C26ACAC9720A1D4FB8D9DC52E95B45ECFEC2D5B5583276A09E8936BC5E">>}, @@ -135,20 +116,17 @@ {<<"prometheus_httpd">>, <<"8F767D819A5D36275EAB9264AFF40D87279151646776069BF69FBDBBD562BD75">>}, {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>}, {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>}, - {<<"recon">>, <<"9052588E83BFEDFD9B72E1034532AEE2A5369D9D9343B61AEB7FBCE761010741">>}, {<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>}, {<<"tls_certificate_check">>, <<"4473005EB0BBDAD215D7083A230E2E076F538D9EA472C8009FD22006A4CFC5F6">>}, {<<"unicode_util_compat">>, <<"A48703A25C170EEDADCA83B11E88985AF08D35F37C6F664D6DCFB106A97782FC">>}]}, {pkg_hash_ext,[ {<<"accept">>, <<"CA69388943F5DAD2E7232A5478F16086E3C872F48E32B88B378E1885A59F5649">>}, {<<"acceptor_pool">>, <<"0CBCD83FDC8B9AD2EEE2067EF8B91A14858A5883CB7CD800E6FCD5803E158788">>}, - {<<"brod">>, <<"88584FDEBA746AA6729E2A1826416C10899954F68AF93659B3C2F38A2DCAA27C">>}, {<<"cache">>, <<"44516CE6FA03594D3A2AF025DD3A87BFE711000EB730219E1DDEFC816E0AA2F4">>}, {<<"certifi">>, <<"6AC7EFC1C6F8600B08D625292D4BBF584E14847CE1B6B5C44D983D273E1097EA">>}, {<<"chatterbox">>, <<"4F75B91451338BC0DA5F52F3480FA6EF6E3A2AEECFC33686D6B3D0A0948F31AA">>}, {<<"cowboy">>, <<"2C729F934B4E1AA149AFF882F57C6372C15399A20D54F65C8D67BEF583021BDE">>}, {<<"cowlib">>, <<"2B3E9DA0B21C4565751A6D4901C20D1B4CC25CBB7FD50D91D2AB6DD287BC86A9">>}, - {<<"crc32cer">>, <<"A39B8F0B1990AC1BF06C3A247FC6A178B740CDFC33C3B53688DC7DD6B1855942">>}, {<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>}, {<<"epgsql">>, <<"B6D86B7DC42C8555B1D4E20880E5099D6D6D053148000E188E548F98E4E01836">>}, {<<"erlydtl">>, <<"D80EC044CD8F58809C19D29AC5605BE09E955040911B644505E31E9DD8143431">>}, @@ -159,7 +137,6 @@ {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, {<<"jsone">>, <<"08560B78624A12E0B5E7EC0271EC8CA38EF51F63D84D84843473E14D9B12618C">>}, {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>}, - {<<"kafka_protocol">>, <<"DF680A3706EAD8695F8B306897C0A33E8063C690DA9308DB87B462CFD7029D04">>}, {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, {<<"mimerl">>, <<"13AF15F9F68C65884ECCA3A3891D50A7B57D82152792F3E19D88650AA126B144">>}, {<<"opentelemetry">>, <<"8E09EDC26AAD11161509D7ECAD854A3285D88580F93B63B0B1CF0BAC332BFCC0">>}, @@ -172,7 +149,6 @@ {<<"prometheus_httpd">>, <<"67736D000745184D5013C58A63E947821AB90CB9320BC2E6AE5D3061C6FFE039">>}, {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>}, {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>}, - {<<"recon">>, <<"96C6799792D735CC0F0FD0F86267E9D351E63339CBE03DF9D162010CEFC26BB0">>}, {<<"ssl_verify_fun">>, <<"FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8">>}, {<<"tls_certificate_check">>, <<"5B0D0E5CB0F928BC4F210DF667304ED91C5BFF2A391CE6BDEDFBFE70A8F096C5">>}, {<<"unicode_util_compat">>, <<"B3A917854CE3AE233619744AD1E0102E05673136776FB2FA76234F3E03B23642">>}]}