Skip to content

Commit

Permalink
Rename external aliases and make use of ext_test_server
Browse files Browse the repository at this point in the history
  • Loading branch information
@Mikaka27
Mikaka27 committed Dec 3, 2024
1 parent 8046d68 commit a03f420
Show file tree
Hide file tree
Showing 5 changed files with 141 additions and 131 deletions.
180 changes: 58 additions & 122 deletions lib/mnesia/test/ext_test.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

-module(ext_test).

-include("ext_test_server.hrl").

%% Initializations
-export([init_backend/0, add_aliases/1, remove_aliases/1,
check_definition/4, semantics/2]).
Expand Down Expand Up @@ -47,29 +49,17 @@
select/1, select/3, select/4, repair_continuation/2
]).

-ifdef(DEBUG).
-define(DBG(DATA), io:format("~p:~p: ~p~n",[?MODULE, ?LINE, DATA])).
-define(DBG(FORMAT, ARGS), io:format("~p:~p: " ++ FORMAT,[?MODULE, ?LINE] ++ ARGS)).
-else.
-define(DBG(DATA), ok).
-define(DBG(FORMAT, ARGS), ok).
-endif.

%% types() ->
%% [{fs_copies, ?MODULE},
%% {raw_fs_copies, ?MODULE}].

semantics(ext_ets, storage) -> ram_copies;
semantics(ext_ets, types ) -> [set, ordered_set, bag];
semantics(ext_ets, index_types) -> [ordered];
semantics(ext_ram_copies, storage) -> ram_copies;
semantics(ext_ram_copies, types ) -> [set, ordered_set, bag];
semantics(ext_ram_copies, index_types) -> [ordered];
semantics(ext_disc_only_copies, storage) -> disc_only_copies;
semantics(ext_disc_only_copies, types ) -> [set, bag];
semantics(ext_disc_only_copies, index_types) -> [bag];
semantics(_Alias, _) ->
undefined.

%% valid_op(_, _) ->
%% true.

init_backend() ->
?DBG(init_backend),
?DBG(),
%% cheat and stuff a marker in mnesia_gvar
K = backend_init_marker(),
case try ets:lookup_element(mnesia_gvar, K, 2) catch _:_ -> error end of
Expand All @@ -80,12 +70,8 @@ init_backend() ->
end,
ok.

backend_init_marker() ->
{test, ?MODULE, backend_init}.

add_aliases(_As) ->
?DBG(_As),
%ct:log("add_aliases(~p)", [_As]),
true = mnesia_lib:val(backend_init_marker()),
ok.

Expand All @@ -95,84 +81,46 @@ remove_aliases(_) ->

%% Table operations

check_definition(ext_ets, _Tab, _Nodes, _Props) ->
?DBG("~p ~p ~p~n", [_Tab, _Nodes, _Props]),
ok.

create_table(ext_ets, Tab, Props) when is_atom(Tab) ->
Tid = ets:new(Tab, [public, proplists:get_value(type, Props, set), {keypos, 2}]),
?DBG("~p Create: ~p(~p) ~p~n", [self(), Tab, Tid, Props]),
mnesia_lib:set({?MODULE, Tab}, Tid),
ok;
create_table(_, Tag={Tab, index, {_Where, Type0}}, _Opts) ->
Type = case Type0 of
ordered -> ordered_set;
_ -> Type0
end,
Tid = ets:new(Tab, [public, Type]),
?DBG("~p(~p) ~p~n", [Tab, Tid, Tag]),
mnesia_lib:set({?MODULE, Tag}, Tid),
check_definition(ext_ram_copies, _Tab, _Nodes, _Props) ->
?DBG("~p ~p ~p~n", [ext_test_server:tab_to_list(_Tab), _Nodes, _Props]),
ok;
create_table(_, Tag={_Tab, retainer, ChkPName}, _Opts) ->
Tid = ets:new(ChkPName, [set, public, {keypos, 2}]),
?DBG("~p(~p) ~p~n", [_Tab, Tid, Tag]),
mnesia_lib:set({?MODULE, Tag}, Tid),
check_definition(ext_disc_only_copies, _Tab, _Nodes, _Props) ->
?DBG("~p ~p ~p~n", [ext_test_server:tab_to_list(_Tab), _Nodes, _Props]),
ok.

delete_table(ext_ets, Tab) ->
try
ets:delete(mnesia_lib:val({?MODULE,Tab})),
mnesia_lib:unset({?MODULE,Tab}),
ok
catch _:_ ->
?DBG({double_delete, Tab}),
ok
end.

load_table(ext_ets, _Tab, init_index, _Cs) -> ok;
load_table(ext_ets, _Tab, _LoadReason, _Cs) ->
?DBG("Load ~p ~p~n", [_Tab, _LoadReason]),
ok.
%% mnesia_monitor:unsafe_create_external(Tab, ext_ets, ?MODULE, Cs).
create_table(Alias, Tab, Props) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Props}).

delete_table(Alias, Tab) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab}).

load_table(Alias, Tab, LoadReason, Cs) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, LoadReason, Cs}).

sender_init(Alias, Tab, _RemoteStorage, _Pid) ->
KeysPerTransfer = 100,
{standard,
fun() -> mnesia_lib:db_init_chunk({ext,Alias,?MODULE}, Tab, KeysPerTransfer) end,
fun(Cont) -> mnesia_lib:db_chunk({ext,Alias,?MODULE}, Cont) end}.
sender_init(Alias, Tab, RemoteStorage, Pid) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, RemoteStorage, Pid}).

receiver_first_message(Sender, {first, Size}, _Alias, Tab) ->
?DBG({first,Size}),
?DBG({first, Size}),
{Size, {Tab, Sender}}.

receive_data(Data, ext_ets, Name, Sender, {Name, Tab, Sender}=State) ->
?DBG({Data,State}),
true = ets:insert(Tab, Data),
{more, State};
receive_data(Data, Alias, Tab, Sender, {Name, Sender}) ->
receive_data(Data, Alias, Tab, Sender, {Name, mnesia_lib:val({?MODULE,Tab}), Sender}).
receive_data(Data, Alias, Name, Sender, State) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Data, Alias, Name, Sender, State}).

receive_done(_Alias, _Tab, _Sender, _State) ->
?DBG({done,_State}),
?DBG({done, _State}),
ok.

close_table(Alias, Tab) -> sync_close_table(Alias, Tab).

sync_close_table(ext_ets, _Tab) ->
?DBG(_Tab).
sync_close_table(Alias, Tab) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab}).

fixtable(ext_ets, Tab, Bool) ->
?DBG({Tab,Bool}),
ets:safe_fixtable(mnesia_lib:val({?MODULE,Tab}), Bool).
fixtable(Alias, Tab, Bool) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Bool}).

info(ext_ets, Tab, Type) ->
?DBG({Tab,Type}),
Tid = mnesia_lib:val({?MODULE,Tab}),
try ets:info(Tid, Type) of
Val -> Val
catch _:_ ->
undefined
end.
info(Alias, Tab, Type) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Type}).

real_suffixes() ->
[".dat"].
Expand All @@ -193,58 +141,46 @@ validate_record(_Alias, _Tab, RecName, Arity, Type, _Obj) ->
validate_key(_Alias, _Tab, RecName, Arity, Type, _Key) ->
{RecName, Arity, Type}.

insert(ext_ets, Tab, Obj) ->
?DBG({Tab,Obj}),
try
ets:insert(mnesia_lib:val({?MODULE,Tab}), Obj),
ok
catch _:Reason ->
io:format("CRASH ~p ~p~n",[Reason, mnesia_lib:val({?MODULE,Tab})])
end.
insert(Alias, Tab, Obj) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Obj}).

lookup(ext_ets, Tab, Key) ->
ets:lookup(mnesia_lib:val({?MODULE,Tab}), Key).
lookup(Alias, Tab, Obj) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Obj}).

delete(ext_ets, Tab, Key) ->
ets:delete(mnesia_lib:val({?MODULE,Tab}), Key).
delete(Alias, Tab, Key) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Key}).

match_delete(ext_ets, Tab, Pat) ->
ets:match_delete(mnesia_lib:val({?MODULE,Tab}), Pat).
match_delete(Alias, Tab, Pat) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Pat}).

first(ext_ets, Tab) ->
ets:first(mnesia_lib:val({?MODULE,Tab})).
first(Alias, Tab) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab}).

last(Alias, Tab) -> first(Alias, Tab).

next(ext_ets, Tab, Key) ->
ets:next(mnesia_lib:val({?MODULE,Tab}), Key).
next(Alias, Tab, Key) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Key}).

prev(Alias, Tab, Key) ->
next(Alias, Tab, Key).

slot(ext_ets, Tab, Pos) ->
ets:slot(mnesia_lib:val({?MODULE,Tab}), Pos).
slot(Alias, Tab, Pos) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Pos}).

update_counter(ext_ets, Tab, C, Val) ->
ets:update_counter(mnesia_lib:val({?MODULE,Tab}), C, Val).
update_counter(Alias, Tab, C, Val) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, C, Val}).

select('$end_of_table' = End) -> End;
select({ext_ets, C}) -> ets:select(C).
select(Continuation) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Continuation}).

select(Alias, Tab, Ms) ->
Res = select(Alias, Tab, Ms, 100000),
select_1(Res).
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Ms}).

select_1('$end_of_table') -> [];
select_1({Acc, C}) ->
case ets:select(C) of
'$end_of_table' -> Acc;
{New, Cont} ->
select_1({New ++ Acc, Cont})
end.

select(ext_ets, Tab, Ms, Limit) when is_integer(Limit); Limit =:= infinity ->
ets:select(mnesia_lib:val({?MODULE,Tab}), Ms, Limit).
select(Alias, Tab, Ms, Limit) ->
gen_server:call(?SERVER, {?FUNCTION_NAME, Alias, Tab, Ms, Limit}).

repair_continuation(Cont, Ms) ->
ets:repair_continuation(Cont, Ms).
gen_server:call(?SERVER, {?FUNCTION_NAME, Cont, Ms}).

backend_init_marker() ->
{test, ext_test, backend_init}.
4 changes: 2 additions & 2 deletions lib/mnesia/test/mnesia_config_test.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -730,8 +730,8 @@ backend_plugin_registration(doc) ->
backend_plugin_registration(Config) when is_list(Config) ->
Nodes = ?acquire_schema(1, [{default_properties, []} | Config]),
?match(ok, mnesia:start()),
?match({atomic,ok}, mnesia:add_backend_type(ext_ets, ext_test)),
?match({atomic,ok}, mnesia:add_backend_type(ext_dets, ext_test)),
?match({atomic,ok}, mnesia:add_backend_type(ext_ram_copies, ext_test)),
?match({atomic,ok}, mnesia:add_backend_type(ext_disc_only_copies, ext_test)),
?verify_mnesia(Nodes, []),
?cleanup(1, Config).

Expand Down
81 changes: 77 additions & 4 deletions lib/mnesia/test/mnesia_external_backend_test.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,13 @@
init_per_group/2, end_per_group/2,
suite/0, all/0, groups/0]).

-export([conversion_from_external_to_disc_copies_results_in_data_loss_after_node_restart/1]).
-export([
conversion_from_external_to_disc_copies_results_in_data_loss_after_node_restart/1,
backup_and_restore_fails_with_external_backend/1
]).

-include("mnesia_test_lib.hrl").
-include_lib("stdlib/include/ms_transform.hrl").

-record(some_rec, {some_id :: atom(), some_int :: number(), some_string :: string()}).

Expand All @@ -15,17 +19,27 @@
delete_schema],
N, Config, ?FILE, ?LINE)).

all() ->
[conversion_from_external_to_disc_copies_results_in_data_loss_after_node_restart].
all() -> [
backup_and_restore_fails_with_external_backend
].

groups() ->
[].

init_per_testcase(Func, Conf) ->
file:delete(ext_test_server:tab_to_filename(table)),
file:delete("bup0.BUP"),
file:delete("bup1.BUP"),
file:delete("bup2.BUP"),
{ok, _} = gen_server:start_link({local, ext_test_server}, ext_test_server, [self()],
[{timeout, infinity}
%%, {debug, [trace]}
]),
mnesia_test_lib:init_per_testcase(Func, Conf).

end_per_testcase(Func, Conf) ->
mnesia_test_lib:end_per_testcase(Func, Conf).
mnesia_test_lib:end_per_testcase(Func, Conf),
ok = gen_server:stop(ext_test_server).

init_per_group(_GroupName, Config) ->
Config.
Expand Down Expand Up @@ -77,3 +91,62 @@ conversion_from_external_to_disc_copies_results_in_data_loss_after_node_restart(
Data = mnesia:activity(transaction, fun() ->
mnesia:match_object(table, #some_rec{_ = '_'}, read) end
).

backup_and_restore_fails_with_external_backend(Config) when is_list(Config) ->
Node = node(),
Data1 = [
#some_rec{some_id = a, some_int = 1, some_string = "1"},
#some_rec{some_id = b, some_int = 2, some_string = "2"},
#some_rec{some_id = c, some_int = 3, some_string = "3"}
],
Data2 = [
#some_rec{some_id = d, some_int = 4, some_string = "4"},
#some_rec{some_id = e, some_int = 5, some_string = "5"},
#some_rec{some_id = f, some_int = 6, some_string = "6"}
],

[Node] = ?acquire_nodes(1, Config),
?match({atomic, ok}, mnesia:create_table(table, [
{type, set},
{record_name, some_rec},
{attributes, record_info(fields, some_rec)},
{ext_disc_only_copies, [Node]}
])),

% ?match({atomic, ok}, mnesia:add_table_index(table, #some_rec.some_int)).
?match([], mnesia:dirty_match_object(table, #some_rec{_ = '_'})),
% ?match(ok, mnesia:backup("bup0.BUP")),

?match(ok, mnesia:activity(transaction, fun() ->
lists:foreach(fun(Elem) -> mnesia:write(table, Elem, write) end, Data1)
end)),
?match(ok, mnesia:backup("bup1.BUP")),

% ?match(ok, mnesia:activity(transaction, fun() ->
% lists:foreach(fun(Elem) -> mnesia:write(table, Elem, write) end, Data2)
% end)),
% ?match(ok, mnesia:backup("bup2.BUP")),

% ?match(true, ets:whereis(table) =/= undefined),
% ?match(ok, load_backup("bup0.BUP")).
% ?match(true, ets:whereis(table) =/= undefined),
% ?match([], mnesia:dirty_match_object(table, #some_rec{_ = '_'})).
% [] = mnesia:dirty_index_read(table, 2, #some_rec.some_int),

?match(ok, load_backup("bup1.BUP")),
Expected1 = sets:from_list(Data1),
?match(Expected1, sets:from_list(mnesia:dirty_match_object(table, #some_rec{_ = '_'}))).
% [#some_rec{some_id = b, some_int = 2, some_string = "2"}] = mnesia:dirty_index_read(table, 2, #some_rec.some_int),

% ?match(ok, load_backup("bup2.BUP")),
% Expected2 = sets:from_list(lists:append(Data1, Data2)),
% ?match(Expected2, sets:from_list(mnesia:dirty_match_object(table, #some_rec{_ = '_'}))).
% [#some_rec{some_id = b, some_int = 2, some_string = "2"}] = mnesia:dirty_index_read(table, 2, #some_rec.some_int),
% [#some_rec{some_id = e, some_int = 5, some_string = "5"}] = mnesia:dirty_index_read(table, 5, #some_rec.some_int).

load_backup(BUP) ->
ok = mnesia:install_fallback(BUP),
stopped = mnesia:stop(),
timer:sleep(3000),
ok = mnesia:start(),
ok = mnesia:wait_for_tables([schema, table], 5000).
5 changes: 3 additions & 2 deletions lib/mnesia/test/mnesia_test_lib.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1031,9 +1031,10 @@ verify_replica_location(Tab, DiscOnly0, Ram0, Disc0, AliveNodes0) ->
timer:sleep(100),

S1 = ?match(AliveNodes, lists:sort(mnesia:system_info(running_db_nodes))),
S2 = ?match(DiscOnly, lists:sort(mnesia:table_info(Tab, disc_only_copies))),
S2 = ?match(DiscOnly, lists:sort(mnesia:table_info(Tab, disc_only_copies) ++
mnesia:table_info(Tab, ext_disc_only_copies))),
S3 = ?match(Ram, lists:sort(mnesia:table_info(Tab, ram_copies) ++
mnesia:table_info(Tab, ext_ets))),
mnesia:table_info(Tab, ext_ram_copies))),
S4 = ?match(Disc, lists:sort(mnesia:table_info(Tab, disc_copies))),
S5 = ?match(Write, lists:sort(mnesia:table_info(Tab, where_to_write))),
S6 = case lists:member(This, Read) of
Expand Down
2 changes: 1 addition & 1 deletion lib/mnesia/test/mnesia_test_lib.hrl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,4 +151,4 @@
-define(verify_mnesia(Ups, Downs),
mnesia_test_lib:verify_mnesia(Ups, Downs, ?FILE, ?LINE)).

-define(BACKEND, [{backend_types, [{ext_ets, ext_test},{ext_dets, ext_test}]}]).
-define(BACKEND, [{backend_types, [{ext_ram_copies, ext_test},{ext_disc_only_copies, ext_test}]}]).

0 comments on commit a03f420

Please sign in to comment.