From 4e259b1b0d5f8b09c995a1c5137fef27ce0e03ea Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Sun, 14 Jan 2024 20:25:22 +0100 Subject: [PATCH 1/4] Add mod_caps_rdbms --- big_tests/tests/disco_and_caps_SUITE.erl | 8 ++- big_tests/tests/gdpr_SUITE.erl | 5 +- big_tests/tests/muc_helper.erl | 2 +- big_tests/tests/pep_SUITE.erl | 8 ++- priv/mssql2012.sql | 7 +++ priv/mysql.sql | 7 +++ priv/pg.sql | 7 +++ src/mod_caps.erl | 68 +++++++----------------- src/mod_caps_backend.erl | 52 ++++++++++++++++++ src/mod_caps_mnesia.erl | 49 +++++++++++++++++ src/mod_caps_rdbms.erl | 45 ++++++++++++++++ test/common/config_parser_helper.erl | 3 +- test/common/mongoose_helper.erl | 6 +-- test/config_parser_SUITE.erl | 1 + 14 files changed, 208 insertions(+), 60 deletions(-) create mode 100644 src/mod_caps_backend.erl create mode 100644 src/mod_caps_mnesia.erl create mode 100644 src/mod_caps_rdbms.erl diff --git a/big_tests/tests/disco_and_caps_SUITE.erl b/big_tests/tests/disco_and_caps_SUITE.erl index 83e16dfb56e..766b59602f9 100644 --- a/big_tests/tests/disco_and_caps_SUITE.erl +++ b/big_tests/tests/disco_and_caps_SUITE.erl @@ -196,10 +196,14 @@ user_can_query_server_info(Config) -> %% Helpers required_modules(disco_with_caps) -> - [{mod_caps, config_parser_helper:default_mod_config(mod_caps)}, + HostType = domain_helper:host_type(), + Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), + [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, {mod_disco, default_mod_config(mod_disco)}]; required_modules(disco_with_caps_and_extra_features) -> - [{mod_caps, config_parser_helper:default_mod_config(mod_caps)}, + HostType = domain_helper:host_type(), + Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), + [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, {mod_disco, mod_config(mod_disco, extra_disco_opts())}]; required_modules(disco_with_extra_features) -> [{mod_disco, mod_config(mod_disco, extra_disco_opts())}]. diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index 8eecfe7eb95..9601a51311e 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -416,7 +416,10 @@ pubsub_required_modules(Plugins) -> host => HostPattern, nodetree => nodetree_tree, plugins => Plugins}), - [{mod_caps, config_parser_helper:default_mod_config(mod_caps)}, {mod_pubsub, PubsubConfig}]. + HostType = domain_helper:host_type(), + Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), + [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, + {mod_pubsub, PubsubConfig}]. is_mim2_started() -> #{node := Node} = distributed_helper:mim2(), diff --git a/big_tests/tests/muc_helper.erl b/big_tests/tests/muc_helper.erl index 783a1d4ad4d..8e0c33745fb 100644 --- a/big_tests/tests/muc_helper.erl +++ b/big_tests/tests/muc_helper.erl @@ -66,7 +66,7 @@ load_muc(HostType) -> hibernated_room_timeout => 2000, access => muc, access_create => muc_create}, LogOpts = #{outdir => "/tmp/muclogs", access_log => muc}, - dynamic_modules:start(HostType, mod_muc, make_opts(Opts)), + Res = dynamic_modules:start(HostType, mod_muc, make_opts(Opts)), dynamic_modules:start(HostType, mod_muc_log, make_log_opts(LogOpts)). unload_muc() -> diff --git a/big_tests/tests/pep_SUITE.erl b/big_tests/tests/pep_SUITE.erl index a18703b284f..a79f6354902 100644 --- a/big_tests/tests/pep_SUITE.erl +++ b/big_tests/tests/pep_SUITE.erl @@ -504,14 +504,18 @@ field_spec({Var, Value}) when is_list(Value) -> #{var => Var, values => Value}; field_spec({Var, Value}) -> #{var => Var, values => [Value]}. required_modules() -> - [{mod_caps, config_parser_helper:default_mod_config(mod_caps)}, + HostType = domain_helper:host_type(), + Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), + [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, {mod_pubsub, mod_config(mod_pubsub, #{plugins => [<<"dag">>, <<"pep">>], nodetree => nodetree_dag, backend => mongoose_helper:mnesia_or_rdbms_backend(), pep_mapping => #{}, host => subhost_pattern("pubsub.@HOST@")})}]. required_modules(cache_tests) -> - [{mod_caps, config_parser_helper:default_mod_config(mod_caps)}, + HostType = domain_helper:host_type(), + Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), + [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, {mod_pubsub, mod_config(mod_pubsub, #{plugins => [<<"dag">>, <<"pep">>], nodetree => nodetree_dag, backend => mongoose_helper:mnesia_or_rdbms_backend(), diff --git a/priv/mssql2012.sql b/priv/mssql2012.sql index 7712a00c172..3ab88ad8dec 100644 --- a/priv/mssql2012.sql +++ b/priv/mssql2012.sql @@ -762,3 +762,10 @@ CREATE TABLE discovery_nodes ( PRIMARY KEY (cluster_name, node_name) ); CREATE UNIQUE INDEX i_discovery_nodes_node_num ON discovery_nodes(cluster_name, node_num); + +CREATE TABLE caps ( + node varchar(250) NOT NULL, + sub_node varchar(250) NOT NULL, + features text NOT NULL, + PRIMARY KEY (node, sub_node) +); diff --git a/priv/mysql.sql b/priv/mysql.sql index 6d77894729c..dacb071e3a8 100644 --- a/priv/mysql.sql +++ b/priv/mysql.sql @@ -554,3 +554,10 @@ CREATE TABLE discovery_nodes ( PRIMARY KEY (cluster_name, node_name) ); CREATE UNIQUE INDEX i_discovery_nodes_node_num USING BTREE ON discovery_nodes(cluster_name, node_num); + +CREATE TABLE caps ( + node varchar(250) NOT NULL, + sub_node varchar(250) NOT NULL, + features text NOT NULL, + PRIMARY KEY (node, sub_node) +); diff --git a/priv/pg.sql b/priv/pg.sql index 8b78cddffbf..6de140cbf03 100644 --- a/priv/pg.sql +++ b/priv/pg.sql @@ -514,3 +514,10 @@ CREATE TABLE discovery_nodes ( PRIMARY KEY (cluster_name, node_name) ); CREATE UNIQUE INDEX i_discovery_nodes_node_num ON discovery_nodes USING BTREE(cluster_name, node_num); + +CREATE TABLE caps ( + node varchar(250) NOT NULL, + sub_node varchar(250) NOT NULL, + features text NOT NULL, + PRIMARY KEY (node, sub_node) +); diff --git a/src/mod_caps.erl b/src/mod_caps.erl index b37d720c253..1398d574ac9 100644 --- a/src/mod_caps.erl +++ b/src/mod_caps.erl @@ -51,8 +51,8 @@ filter_pep_recipient/3]). %% for test cases --export([delete_caps/1, make_disco_hash/2]). --ignore_xref([delete_caps/1, make_disco_hash/2, read_caps/1, start_link/2]). +-export([delete_caps/2, make_disco_hash/2]). +-ignore_xref([delete_caps/2, make_disco_hash/2, read_caps/1, start_link/2]). -include("mongoose.hrl"). -include("mongoose_config_spec.hrl"). @@ -74,17 +74,11 @@ -type caps() :: #caps{}. -type caps_resources() :: gb_trees:tree(jid:simple_jid(), caps()). --export_type([caps/0]). +-export_type([caps/0, node_pair/0, maybe_pending_features/0]). -type features() :: [binary()]. -type maybe_pending_features() :: features() | pos_integer(). --type node_pair() :: {binary(), binary()}. - --record(caps_features, - { - node_pair = {<<>>, <<>>} :: node_pair(), - features = [] :: maybe_pending_features() - }). +-type node_pair() :: {Node :: binary(), SubNode :: binary()}. -record(state, {host_type :: mongooseim:host_type()}). @@ -92,6 +86,7 @@ -spec start_link(mongooseim:host_type(), gen_mod:module_opts()) -> any(). start_link(HostType, Opts) -> + mod_caps_backend:init(HostType, Opts), Proc = gen_mod:get_module_proc(HostType, ?PROCNAME), gen_server:start_link({local, Proc}, ?MODULE, [HostType, Opts], []). @@ -115,10 +110,13 @@ config_spec() -> items = #{<<"cache_size">> => #option{type = integer, validate = positive}, <<"cache_life_time">> => #option{type = integer, - validate = positive} + validate = positive}, + <<"backend">> => #option{type = atom, + validate = {module, ?MODULE}} }, defaults = #{<<"cache_size">> => 1000, - <<"cache_life_time">> => timer:hours(24) div 1000} + <<"cache_life_time">> => timer:hours(24) div 1000, + <<"backend">> => mnesia} }. supported_features() -> [dynamic_domains]. @@ -369,23 +367,8 @@ filter_pep_recipient(InAcc, #{c2s_data := C2SData, feature := Feature, to := To} _ -> {ok, InAcc} end. -init_db(mnesia) -> - case catch mnesia:table_info(caps_features, storage_type) of - {'EXIT', _} -> - ok; - disc_only_copies -> - ok; - _ -> - mnesia:delete_table(caps_features) - end, - mongoose_mnesia:create_table(caps_features, - [{disc_only_copies, [node()]}, - {local_content, true}, - {attributes, record_info(fields, caps_features)}]). - -spec init(list()) -> {ok, state()}. init([HostType, #{cache_size := MaxSize, cache_life_time := LifeTime}]) -> - init_db(db_type(HostType)), cache_tab:new(caps_features, [{max_size, MaxSize}, {life_time, LifeTime}]), gen_hook:add_handlers(hooks(HostType)), {ok, #state{host_type = HostType}}. @@ -493,37 +476,25 @@ feature_response(Acc, _IQResult, LServer, From, Caps, [_SubNode | SubNodes]) -> -spec caps_read_fun(mongooseim:host_type(), node_pair()) -> fun(() -> {ok, maybe_pending_features()} | error). caps_read_fun(HostType, Node) -> - DBType = db_type(HostType), - caps_read_fun(HostType, Node, DBType). - -caps_read_fun(_HostType, Node, mnesia) -> fun () -> - case mnesia:dirty_read({caps_features, Node}) of - [#caps_features{features = Features}] -> {ok, Features}; - _ -> error - end + mod_caps_backend:read(HostType, Node) end. -spec caps_write_fun(mongooseim:host_type(), node_pair(), maybe_pending_features()) -> fun(() -> ok). caps_write_fun(HostType, Node, Features) -> - DBType = db_type(HostType), - caps_write_fun(HostType, Node, Features, DBType). - -caps_write_fun(_HostType, Node, Features, mnesia) -> fun () -> - mnesia:dirty_write(#caps_features{node_pair = Node, - features = Features}) + mod_caps_backend:write(HostType, Node, Features) end. --spec delete_caps(node_pair()) -> ok. -delete_caps(Node) -> - cache_tab:delete(caps_features, Node, caps_delete_fun(Node)). +-spec delete_caps(mongooseim:host_type(), node_pair()) -> ok. +delete_caps(HostType, Node) -> + cache_tab:delete(caps_features, Node, caps_delete_fun(HostType, Node)). --spec caps_delete_fun(node_pair()) -> fun(() -> ok). -caps_delete_fun(Node) -> +-spec caps_delete_fun(mongooseim:host_type(), node_pair()) -> fun(() -> ok). +caps_delete_fun(HostType, Node) -> fun () -> - mnesia:dirty_delete(caps_features, Node) + mod_caps_backend:delete_node(HostType, Node) end. -spec make_my_disco_hash(mongooseim:host_type(), jid:lserver()) -> binary(). @@ -625,6 +596,3 @@ is_valid_node(Node) -> _ -> false end. - -db_type(_HostType) -> - mnesia. diff --git a/src/mod_caps_backend.erl b/src/mod_caps_backend.erl new file mode 100644 index 00000000000..6b58bd6e050 --- /dev/null +++ b/src/mod_caps_backend.erl @@ -0,0 +1,52 @@ +-module(mod_caps_backend). +-export([init/2, + read/2, + write/3, + delete_node/2]). + +-define(MAIN_MODULE, mod_caps). + +%% ---------------------------------------------------------------------- +%% Callbacks +%% (exactly the same as specs in this module) + +-callback init(HostType, Opts) -> ok when + HostType :: mongooseim:host_type(), + Opts :: gen_mod:module_opts(). + +-callback read(mongooseim:host_type(), mod_caps:node_pair()) -> + {ok, mod_caps:maybe_pending_features()} | error. + +-callback write(mongooseim:host_type(), mod_caps:node_pair(), + mod_caps:maybe_pending_features()) -> ok. + +-callback delete_node(mongooseim:host_type(), mod_caps:node_pair()) -> ok. + +%% ---------------------------------------------------------------------- +%% API Functions + +-spec init(HostType, Opts) -> ok when + HostType :: mongooseim:host_type(), + Opts :: gen_mod:module_opts(). +init(HostType, Opts) -> + TrackedFuns = [], + mongoose_backend:init(HostType, ?MAIN_MODULE, TrackedFuns, Opts), + Args = [HostType, Opts], + mongoose_backend:call(HostType, ?MAIN_MODULE, ?FUNCTION_NAME, Args). + +-spec read(mongooseim:host_type(), mod_caps:node_pair()) -> + {ok, mod_caps:maybe_pending_features()} | error. +read(HostType, Node) -> + Args = [HostType, Node], + mongoose_backend:call_tracked(HostType, ?MAIN_MODULE, ?FUNCTION_NAME, Args). + +-spec write(mongooseim:host_type(), mod_caps:node_pair(), + mod_caps:maybe_pending_features()) -> ok. +write(HostType, Node, Features) -> + Args = [HostType, Node, Features], + mongoose_backend:call_tracked(HostType, ?MAIN_MODULE, ?FUNCTION_NAME, Args). + +-spec delete_node(mongooseim:host_type(), mod_caps:node_pair()) -> ok. +delete_node(HostType, Node) -> + Args = [HostType, Node], + mongoose_backend:call_tracked(HostType, ?MAIN_MODULE, ?FUNCTION_NAME, Args). diff --git a/src/mod_caps_mnesia.erl b/src/mod_caps_mnesia.erl new file mode 100644 index 00000000000..30939d26b58 --- /dev/null +++ b/src/mod_caps_mnesia.erl @@ -0,0 +1,49 @@ +-module(mod_caps_mnesia). +-behaviour(mod_caps_backend). + +-export([init/2, + read/2, + write/3, + delete_node/2]). + +-record(caps_features, + { + node_pair = {<<>>, <<>>} :: mod_caps:node_pair(), + features = [] :: mod_caps:maybe_pending_features() + }). + +-spec init(mongooseim:host_type(), gen_mod:module_opts()) -> ok. +init(_HostName, _Opts) -> + case catch mnesia:table_info(caps_features, storage_type) of + {'EXIT', _} -> + ok; + disc_only_copies -> + ok; + _ -> + mnesia:delete_table(caps_features) + end, + mongoose_mnesia:create_table(caps_features, + [{disc_only_copies, [node()]}, + {local_content, true}, + {attributes, record_info(fields, caps_features)}]), + ok. + +-spec read(mongooseim:host_type(), mod_caps:node_pair()) -> + {ok, mod_caps:maybe_pending_features()} | error. +read(_HostType, Node) -> + case mnesia:dirty_read({caps_features, Node}) of + [#caps_features{features = Features}] -> {ok, Features}; + _ -> error + end. + +-spec write(mongooseim:host_type(), mod_caps:node_pair(), + mod_caps:maybe_pending_features()) -> ok. +write(_HostType, Node, Features) -> + mnesia:dirty_write(#caps_features{node_pair = Node, + features = Features}), + ok. + +-spec delete_node(mongooseim:host_type(), mod_caps:node_pair()) -> ok. +delete_node(_HostType, Node) -> + mnesia:dirty_delete(caps_features, Node), + ok. diff --git a/src/mod_caps_rdbms.erl b/src/mod_caps_rdbms.erl new file mode 100644 index 00000000000..4baf42a9e7e --- /dev/null +++ b/src/mod_caps_rdbms.erl @@ -0,0 +1,45 @@ +-module(mod_caps_rdbms). +-behaviour(mod_caps_backend). + +-export([init/2, + read/2, + write/3, + delete_node/2]). + +-spec init(mongooseim:host_type(), gen_mod:module_opts()) -> ok. +init(HostType, _Opts) -> + rdbms_queries:prepare_upsert(HostType, caps_upsert, caps, + [<<"node">>, <<"sub_node">>, <<"features">>], + [<<"features">>], + [<<"node">>, <<"sub_node">>]), + mongoose_rdbms:prepare(caps_delete, caps, + [node, sub_node], + <<"DELETE FROM caps WHERE node=? AND sub_node=?">>), + mongoose_rdbms:prepare(caps_select, caps, + [node, sub_node], + <<"SELECT features FROM caps WHERE node=? AND sub_node=?">>), + ok. + +-spec read(mongooseim:host_type(), mod_caps:node_pair()) -> + {ok, mod_caps:maybe_pending_features()} | error. +read(HostType, {Node, SubNode}) -> + case mongoose_rdbms:execute(HostType, caps_select, [Node, SubNode]) of + [{selected, [{Encoded}]}] -> {ok, jiffy:decode(Encoded)}; + _ -> error + end. + +-spec write(mongooseim:host_type(), mod_caps:node_pair(), + mod_caps:maybe_pending_features()) -> ok. +write(HostType, {Node, SubNode}, Features) -> + Encoded = jiffy:encode(Features), + InsertParams = [Node, SubNode, Encoded], + UpdateParams = [Encoded], + UniqueKeyValues = [Node, SubNode], + rdbms_queries:execute_upsert(HostType, caps_upsert, + InsertParams, UpdateParams, UniqueKeyValues), + ok. + +-spec delete_node(mongooseim:host_type(), mod_caps:node_pair()) -> ok. +delete_node(HostType, {Node, SubNode}) -> + {updated, _} = mongoose_rdbms:execute(HostType, caps_delete, [Node, SubNode]), + ok. diff --git a/test/common/config_parser_helper.erl b/test/common/config_parser_helper.erl index acd77f3b24b..10650a9e30b 100644 --- a/test/common/config_parser_helper.erl +++ b/test/common/config_parser_helper.erl @@ -838,7 +838,8 @@ default_mod_config(mod_bosh) -> default_mod_config(mod_cache_users) -> #{strategy => fifo, time_to_live => 480, number_of_segments => 3}; default_mod_config(mod_caps) -> - #{cache_size => 1000, + #{backend => mnesia, + cache_size => 1000, cache_life_time => timer:hours(24) div 1000}; default_mod_config(mod_csi) -> #{buffer_max => 20}; diff --git a/test/common/mongoose_helper.erl b/test/common/mongoose_helper.erl index 867c1f7b0a3..a5bda0d3063 100644 --- a/test/common/mongoose_helper.erl +++ b/test/common/mongoose_helper.erl @@ -23,7 +23,7 @@ generic_count/1]). -export([clear_last_activity/2, - clear_caps_cache/1]). + clear_caps_cache/2]). -export([kick_everyone/0]). -export([ensure_muc_clean/0]). @@ -154,8 +154,8 @@ new_mongoose_acc(Location, Server) -> host_type => HostType, element => undefined }]). -clear_caps_cache(CapsNode) -> - ok = rpc(mim(), mod_caps, delete_caps, [CapsNode]). +clear_caps_cache(HostType, CapsNode) -> + ok = rpc(mim(), mod_caps, delete_caps, [HostType, CapsNode]). get_backend(HostType, Module) -> try rpc(mim(), mongoose_backend, get_backend_module, [HostType, Module]) diff --git a/test/config_parser_SUITE.erl b/test/config_parser_SUITE.erl index df06350058e..c740a30d927 100644 --- a/test/config_parser_SUITE.erl +++ b/test/config_parser_SUITE.erl @@ -1457,6 +1457,7 @@ mod_caps(_Config) -> P = [modules, mod_caps], ?cfgh(P ++ [cache_size], 10, T(<<"cache_size">>, 10)), ?cfgh(P ++ [cache_life_time], 10, T(<<"cache_life_time">>, 10)), + ?cfgh(P ++ [backend], mnesia, T(<<"backend">>, <<"mnesia">>)), ?errh(T(<<"cache_size">>, 0)), ?errh(T(<<"cache_size">>, <<"infinity">>)), ?errh(T(<<"cache_life_time">>, 0)), From ad6c993bae7d83e19eefe1ad520546a321460dcc Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 15 Jan 2024 15:15:09 +0100 Subject: [PATCH 2/4] Use mod_config_with_auto_backend/1 to reduce code duplication --- big_tests/tests/disco_and_caps_SUITE.erl | 10 +++------- big_tests/tests/gdpr_SUITE.erl | 4 +--- big_tests/tests/graphql_offline_SUITE.erl | 14 ++++++-------- big_tests/tests/graphql_private_SUITE.erl | 13 ++++++------- big_tests/tests/last_SUITE.erl | 12 ++++++------ big_tests/tests/mod_blocking_SUITE.erl | 11 ++++------- big_tests/tests/mod_event_pusher_http_SUITE.erl | 11 +++++------ big_tests/tests/offline_SUITE.erl | 17 +++++++---------- big_tests/tests/pep_SUITE.erl | 7 ++----- big_tests/tests/privacy_SUITE.erl | 10 ++-------- big_tests/tests/private_SUITE.erl | 16 +++++++--------- test/common/config_parser_helper.erl | 9 +++++++++ 12 files changed, 58 insertions(+), 76 deletions(-) diff --git a/big_tests/tests/disco_and_caps_SUITE.erl b/big_tests/tests/disco_and_caps_SUITE.erl index 766b59602f9..731668e1a0c 100644 --- a/big_tests/tests/disco_and_caps_SUITE.erl +++ b/big_tests/tests/disco_and_caps_SUITE.erl @@ -5,7 +5,7 @@ -include_lib("escalus/include/escalus_xmlns.hrl"). -import(domain_helper, [host_type/0, domain/0]). --import(config_parser_helper, [default_mod_config/1, mod_config/2]). +-import(config_parser_helper, [default_mod_config/1, mod_config/2, mod_config_with_auto_backend/1]). all() -> [{group, disco_with_caps}, @@ -196,14 +196,10 @@ user_can_query_server_info(Config) -> %% Helpers required_modules(disco_with_caps) -> - HostType = domain_helper:host_type(), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, + [{mod_caps, mod_config_with_auto_backend(mod_caps)}, {mod_disco, default_mod_config(mod_disco)}]; required_modules(disco_with_caps_and_extra_features) -> - HostType = domain_helper:host_type(), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, + [{mod_caps, mod_config_with_auto_backend(mod_caps)}, {mod_disco, mod_config(mod_disco, extra_disco_opts())}]; required_modules(disco_with_extra_features) -> [{mod_disco, mod_config(mod_disco, extra_disco_opts())}]. diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index 9601a51311e..3794e4db724 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -416,9 +416,7 @@ pubsub_required_modules(Plugins) -> host => HostPattern, nodetree => nodetree_tree, plugins => Plugins}), - HostType = domain_helper:host_type(), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, + [{mod_caps, config_parser_helper:mod_config_with_auto_backend(mod_caps)}, {mod_pubsub, PubsubConfig}]. is_mim2_started() -> diff --git a/big_tests/tests/graphql_offline_SUITE.erl b/big_tests/tests/graphql_offline_SUITE.erl index 6100a704651..388f1db7e84 100644 --- a/big_tests/tests/graphql_offline_SUITE.erl +++ b/big_tests/tests/graphql_offline_SUITE.erl @@ -7,7 +7,7 @@ -import(domain_helper, [host_type/0, domain/0]). -import(graphql_helper, [execute_command/4, get_ok_value/2, get_err_code/1, user_to_bin/1, get_unauthorized/1, get_not_loaded/1, get_coercion_err_msg/1]). --import(config_parser_helper, [mod_config/2]). +-import(config_parser_helper, [mod_config_with_auto_backend/1]). -import(mongooseimctl_helper, [mongooseimctl/3, rpc_call/3]). -include_lib("eunit/include/eunit.hrl"). @@ -70,9 +70,9 @@ init_per_suite(Config) -> Config2 = ejabberd_node_utils:init(mim(), Config1), escalus:init_per_suite(Config2). --spec create_config(atom()) -> [{mod_offline, gen_mod:module_opts()}]. -create_config(Backend) -> - [{mod_offline, mod_config(mod_offline, #{backend => Backend})}]. +-spec create_config() -> [{mod_offline, gen_mod:module_opts()}]. +create_config() -> + [{mod_offline, mod_config_with_auto_backend(mod_offline)}]. end_per_suite(Config) -> dynamic_modules:restore_modules(Config), @@ -87,10 +87,8 @@ init_per_group(domain_admin, Config) -> init_per_group(GroupName, Config) when GroupName =:= admin_offline; GroupName =:= domain_admin_offline -> HostType = host_type(), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - ModConfig = create_config(Backend), - dynamic_modules:ensure_modules(HostType, ModConfig), - [{backend, Backend} | escalus:init_per_suite(Config)]; + dynamic_modules:ensure_modules(HostType, create_config()), + escalus:init_per_suite(Config); init_per_group(admin_offline_not_configured, Config) -> dynamic_modules:ensure_modules(host_type(), [{mod_offline, stopped}]), Config; diff --git a/big_tests/tests/graphql_private_SUITE.erl b/big_tests/tests/graphql_private_SUITE.erl index c956e68f27d..dfcf551fb7b 100644 --- a/big_tests/tests/graphql_private_SUITE.erl +++ b/big_tests/tests/graphql_private_SUITE.erl @@ -5,6 +5,7 @@ -import(distributed_helper, [mim/0, require_rpc_nodes/1]). -import(graphql_helper, [execute_user_command/5, execute_command/4, get_ok_value/2, get_err_code/1, user_to_bin/1, get_unauthorized/1, get_not_loaded/1, get_coercion_err_msg/1]). +-import(config_parser_helper, [mod_config_with_auto_backend/2]). -include_lib("eunit/include/eunit.hrl"). -include_lib("exml/include/exml.hrl"). @@ -67,11 +68,11 @@ init_per_suite(Config0) -> HostType = domain_helper:host_type(), Config1 = dynamic_modules:save_modules(HostType, Config0), Config2 = ejabberd_node_utils:init(mim(), Config1), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - escalus:init_per_suite([{backend, Backend} | Config2]). + escalus:init_per_suite(Config2). -create_config(Backend) -> - [{mod_private, #{backend => Backend, iqdisc => one_queue}}]. +create_config() -> + [{mod_private, + mod_config_with_auto_backend(mod_private, #{iqdisc => one_queue})}]. end_per_suite(Config) -> dynamic_modules:restore_modules(Config), @@ -95,9 +96,7 @@ init_per_group(Group, Config) when Group =:= admin_private_not_configured; ensure_private_started(Config) -> HostType = domain_helper:host_type(), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - ModConfig = create_config(Backend), - dynamic_modules:ensure_modules(HostType, ModConfig), + dynamic_modules:ensure_modules(HostType, create_config()), Config. ensure_private_stopped(Config) -> diff --git a/big_tests/tests/last_SUITE.erl b/big_tests/tests/last_SUITE.erl index a762ea05fc9..55487fea8c7 100644 --- a/big_tests/tests/last_SUITE.erl +++ b/big_tests/tests/last_SUITE.erl @@ -20,6 +20,8 @@ -include_lib("escalus/include/escalus_xmlns.hrl"). -include_lib("exml/include/exml.hrl"). +-import(config_parser_helper, [mod_config_with_auto_backend/2]). + %%-------------------------------------------------------------------- %% Suite configuration %%-------------------------------------------------------------------- @@ -45,9 +47,8 @@ suite() -> init_per_suite(Config0) -> HostType = domain_helper:host_type(), Config1 = dynamic_modules:save_modules(HostType, Config0), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - dynamic_modules:ensure_modules(HostType, required_modules(Backend)), - escalus:init_per_suite([{backend, Backend} | Config1]). + dynamic_modules:ensure_modules(HostType, required_modules()), + escalus:init_per_suite(Config1). end_per_suite(Config) -> dynamic_modules:restore_modules(Config), @@ -190,6 +191,5 @@ answer_last_activity(IQ = #xmlel{name = <<"iq">>}) -> {<<"seconds">>, <<"0">>}]} ]}. -required_modules(Backend) -> - [{mod_last, #{backend => Backend, - iqdisc => one_queue}}]. +required_modules() -> + [{mod_last, mod_config_with_auto_backend(mod_last, #{iqdisc => one_queue})}]. diff --git a/big_tests/tests/mod_blocking_SUITE.erl b/big_tests/tests/mod_blocking_SUITE.erl index d0f2268c175..94b0a9da55f 100644 --- a/big_tests/tests/mod_blocking_SUITE.erl +++ b/big_tests/tests/mod_blocking_SUITE.erl @@ -23,6 +23,8 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("escalus/include/escalus_xmlns.hrl"). +-import(config_parser_helper, [mod_config_with_auto_backend/1]). + -define(SLEEP_TIME, 50). %%-------------------------------------------------------------------- @@ -99,14 +101,9 @@ suite() -> init_per_suite(Config0) -> HostType = domain_helper:host_type(), Config1 = dynamic_modules:save_modules(HostType, Config0), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - ModConfig = [{mod_blocking, set_opts(Backend)}], + ModConfig = [{mod_blocking, mod_config_with_auto_backend(mod_blocking)}], dynamic_modules:ensure_modules(HostType, ModConfig), - [{backend, Backend} | - escalus:init_per_suite(Config1)]. - -set_opts(Backend) -> - #{backend => Backend}. + escalus:init_per_suite(Config1). end_per_suite(Config) -> escalus_fresh:clean(), diff --git a/big_tests/tests/mod_event_pusher_http_SUITE.erl b/big_tests/tests/mod_event_pusher_http_SUITE.erl index ff8e7f09d69..a0fffa5d655 100644 --- a/big_tests/tests/mod_event_pusher_http_SUITE.erl +++ b/big_tests/tests/mod_event_pusher_http_SUITE.erl @@ -22,7 +22,7 @@ -import(domain_helper, [host_type/0]). --import(config_parser_helper, [config/2, mod_config/2, mod_event_pusher_http_handler/0]). +-import(config_parser_helper, [config/2, mod_config_with_auto_backend/1, mod_event_pusher_http_handler/0]). %%%=================================================================== %%% Suite configuration @@ -178,16 +178,15 @@ stop_pool() -> set_modules(Config0, ExtraHandlerOpts) -> Config = dynamic_modules:save_modules(host_type(), Config0), - Backend = mongoose_helper:get_backend_mnesia_rdbms(host_type()), - ModOffline = create_offline_config(Backend), + ModOffline = create_offline_config(), Handler = maps:merge(mod_event_pusher_http_handler(), ExtraHandlerOpts), ModOpts = #{http => #{handlers => [Handler]}}, dynamic_modules:ensure_modules(host_type(), [{mod_event_pusher, ModOpts} | ModOffline]), Config. --spec create_offline_config(atom()) -> [{mod_offline, gen_mod:module_opts()}]. -create_offline_config(Backend) -> - [{mod_offline, mod_config(mod_offline, #{backend => Backend})}]. +-spec create_offline_config() -> [{mod_offline, gen_mod:module_opts()}]. +create_offline_config() -> + [{mod_offline, mod_config_with_auto_backend(mod_offline)}]. start_http_listener(simple_message, Prefix) -> http_helper:start(http_notifications_port(), Prefix, fun process_notification/1); diff --git a/big_tests/tests/offline_SUITE.erl b/big_tests/tests/offline_SUITE.erl index 7f40b71d11f..b7e4d8bae5c 100644 --- a/big_tests/tests/offline_SUITE.erl +++ b/big_tests/tests/offline_SUITE.erl @@ -19,7 +19,7 @@ -import(domain_helper, [host_type/0]). -import(mongoose_helper, [wait_for_n_offline_messages/2]). --import(config_parser_helper, [mod_config/2]). +-import(config_parser_helper, [mod_config/2, mod_config_with_auto_backend/1]). %%%=================================================================== %%% Suite configuration @@ -60,15 +60,12 @@ suite() -> init_per_suite(Config0) -> HostType = domain_helper:host_type(), Config1 = dynamic_modules:save_modules(HostType, Config0), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - ModConfig = create_config(Backend), - dynamic_modules:ensure_modules(HostType, ModConfig), - [{backend, Backend} | - escalus:init_per_suite(Config1)]. - --spec create_config(atom()) -> [{mod_offline, gen_mod:module_opts()}]. -create_config(Backend) -> - [{mod_offline, mod_config(mod_offline, #{backend => Backend})}]. + dynamic_modules:ensure_modules(HostType, create_config()), + escalus:init_per_suite(Config1). + +-spec create_config() -> [{mod_offline, gen_mod:module_opts()}]. +create_config() -> + [{mod_offline, mod_config_with_auto_backend(mod_offline)}]. end_per_suite(Config) -> escalus_fresh:clean(), diff --git a/big_tests/tests/pep_SUITE.erl b/big_tests/tests/pep_SUITE.erl index a79f6354902..afab8d177a8 100644 --- a/big_tests/tests/pep_SUITE.erl +++ b/big_tests/tests/pep_SUITE.erl @@ -504,9 +504,7 @@ field_spec({Var, Value}) when is_list(Value) -> #{var => Var, values => Value}; field_spec({Var, Value}) -> #{var => Var, values => [Value]}. required_modules() -> - HostType = domain_helper:host_type(), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, + [{mod_caps, config_parser_helper:mod_config_with_auto_backend(mod_caps)}, {mod_pubsub, mod_config(mod_pubsub, #{plugins => [<<"dag">>, <<"pep">>], nodetree => nodetree_dag, backend => mongoose_helper:mnesia_or_rdbms_backend(), @@ -514,8 +512,7 @@ required_modules() -> host => subhost_pattern("pubsub.@HOST@")})}]. required_modules(cache_tests) -> HostType = domain_helper:host_type(), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - [{mod_caps, config_parser_helper:mod_config(mod_caps, #{backend => Backend})}, + [{mod_caps, config_parser_helper:mod_config_with_auto_backend(mod_caps)}, {mod_pubsub, mod_config(mod_pubsub, #{plugins => [<<"dag">>, <<"pep">>], nodetree => nodetree_dag, backend => mongoose_helper:mnesia_or_rdbms_backend(), diff --git a/big_tests/tests/privacy_SUITE.erl b/big_tests/tests/privacy_SUITE.erl index 69bd83f9910..188c8c5482f 100644 --- a/big_tests/tests/privacy_SUITE.erl +++ b/big_tests/tests/privacy_SUITE.erl @@ -93,15 +93,9 @@ suite() -> init_per_suite(Config0) -> HostType = domain_helper:host_type(), Config1 = dynamic_modules:save_modules(HostType, Config0), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - ModConfig = [{mod_privacy, set_opts(Backend)}], + ModConfig = [{mod_privacy, config_parser_helper:mod_config_with_auto_backend(mod_privacy)}], dynamic_modules:ensure_modules(HostType, ModConfig), - [{escalus_no_stanzas_after_story, true}, - {backend, Backend} | - escalus:init_per_suite(Config1)]. - -set_opts(Backend) -> - config_parser_helper:mod_config(mod_privacy, #{backend => Backend}). + [{escalus_no_stanzas_after_story, true} | escalus:init_per_suite(Config1)]. end_per_suite(Config) -> escalus_fresh:clean(), diff --git a/big_tests/tests/private_SUITE.erl b/big_tests/tests/private_SUITE.erl index 6260675cff7..95927a17eb6 100644 --- a/big_tests/tests/private_SUITE.erl +++ b/big_tests/tests/private_SUITE.erl @@ -20,6 +20,8 @@ -include_lib("escalus/include/escalus.hrl"). -include_lib("common_test/include/ct.hrl"). +-import(config_parser_helper, [mod_config_with_auto_backend/2]). + %%-------------------------------------------------------------------- %% Suite configuration %%-------------------------------------------------------------------- @@ -47,13 +49,12 @@ suite() -> init_per_suite(Config0) -> HostType = domain_helper:host_type(), Config1 = dynamic_modules:save_modules(HostType, Config0), - Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), - ModConfig = create_config(Backend), - dynamic_modules:ensure_modules(HostType, ModConfig), - escalus:init_per_suite([{backend, Backend} | Config1]). + dynamic_modules:ensure_modules(HostType, create_config()), + escalus:init_per_suite(Config1). -create_config(Backend) -> - [{mod_private, #{backend => Backend, iqdisc => one_queue}}]. +create_config() -> + [{mod_private, + mod_config_with_auto_backend(mod_private, #{iqdisc => one_queue})}]. end_per_suite(Config) -> dynamic_modules:restore_modules(Config), @@ -175,6 +176,3 @@ check_body_rec(Element, [Name | Names]) -> [Child] = Element#xmlel.children, Name = Child#xmlel.name, check_body_rec(Child, Names). - -required_modules(Backend) -> - [{mod_private, [{backend, Backend}]}]. diff --git a/test/common/config_parser_helper.erl b/test/common/config_parser_helper.erl index 10650a9e30b..f63a2b72627 100644 --- a/test/common/config_parser_helper.erl +++ b/test/common/config_parser_helper.erl @@ -822,6 +822,15 @@ default_pool_conn_opts(_Type) -> mod_config(Module, ExtraOpts) -> maps:merge(default_mod_config(Module), ExtraOpts). +%% Selects backend automatically depending on which databases are available +mod_config_with_auto_backend(Module) -> + mod_config_with_auto_backend(Module, #{}). + +mod_config_with_auto_backend(Module, ExtraOpts) -> + HostType = domain_helper:host_type(), + Backend = mongoose_helper:get_backend_mnesia_rdbms(HostType), + mod_config(Module, ExtraOpts#{backend => Backend}). + default_mod_config(mod_adhoc) -> #{iqdisc => one_queue, report_commands_node => false}; default_mod_config(mod_auth_token) -> From caeb37e3ebc1c17a9f3019a29a4cf7e2b0b83267 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 15 Jan 2024 15:19:36 +0100 Subject: [PATCH 3/4] Add docs for modules.mod_caps.backend option --- doc/modules/mod_caps.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/modules/mod_caps.md b/doc/modules/mod_caps.md index 505d4a72e5e..655babc6980 100644 --- a/doc/modules/mod_caps.md +++ b/doc/modules/mod_caps.md @@ -21,6 +21,11 @@ The size of a cache_tab (the amount of entries) holding the information about ca Time (in seconds) after which entries will be removed. +### `modules.mod_caps.backend` +* **Syntax:** string, one of `"mnesia"`, `"rdbms"`. +* **Default:** "mnesia" +* **Example:** `backend = "mnesia"` + ## Example Configuration ```toml From eba748be249b1093c6181f3592d4370b1c531c65 Mon Sep 17 00:00:00 2001 From: Mikhail Uvarov Date: Mon, 15 Jan 2024 15:23:44 +0100 Subject: [PATCH 4/4] Address review comments --- big_tests/tests/muc_helper.erl | 2 +- src/mod_caps.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/big_tests/tests/muc_helper.erl b/big_tests/tests/muc_helper.erl index 8e0c33745fb..783a1d4ad4d 100644 --- a/big_tests/tests/muc_helper.erl +++ b/big_tests/tests/muc_helper.erl @@ -66,7 +66,7 @@ load_muc(HostType) -> hibernated_room_timeout => 2000, access => muc, access_create => muc_create}, LogOpts = #{outdir => "/tmp/muclogs", access_log => muc}, - Res = dynamic_modules:start(HostType, mod_muc, make_opts(Opts)), + dynamic_modules:start(HostType, mod_muc, make_opts(Opts)), dynamic_modules:start(HostType, mod_muc_log, make_log_opts(LogOpts)). unload_muc() -> diff --git a/src/mod_caps.erl b/src/mod_caps.erl index 1398d574ac9..dad980b250a 100644 --- a/src/mod_caps.erl +++ b/src/mod_caps.erl @@ -111,8 +111,8 @@ config_spec() -> validate = positive}, <<"cache_life_time">> => #option{type = integer, validate = positive}, - <<"backend">> => #option{type = atom, - validate = {module, ?MODULE}} + <<"backend">> => #option{type = atom, + validate = {module, ?MODULE}} }, defaults = #{<<"cache_size">> => 1000, <<"cache_life_time">> => timer:hours(24) div 1000,