diff --git a/lib/MirrorCache/WebAPI/Controller/Rest/ReportMirror.pm b/lib/MirrorCache/WebAPI/Controller/Rest/ReportMirror.pm index a3f58eb0..8ebdc882 100644 --- a/lib/MirrorCache/WebAPI/Controller/Rest/ReportMirror.pm +++ b/lib/MirrorCache/WebAPI/Controller/Rest/ReportMirror.pm @@ -91,7 +91,7 @@ sub list { return unless $self->mcproject->caching; if (!$last_cache_run || 60 < time() - $last_cache_run) { - my $wassaveerror = 1; + $last_cache_run = time(); eval { my $f = Mojo::File->new( $self->mcproject->cache_dir . "/$cache_filename.json" ); $f->spurt($body) if $body; @@ -102,9 +102,8 @@ sub list { $f = Mojo::File->new( $self->mcproject->cache_dir . "/$cache_filename$region.json" ); $f->spurt($x); } - $wassaveerror = 0; 1; - } or $last_cache_run = time(); + } } } diff --git a/lib/MirrorCache/WebAPI/Plugin/Project.pm b/lib/MirrorCache/WebAPI/Plugin/Project.pm index 0c8766ec..a93f77ec 100644 --- a/lib/MirrorCache/WebAPI/Plugin/Project.pm +++ b/lib/MirrorCache/WebAPI/Plugin/Project.pm @@ -16,7 +16,8 @@ package MirrorCache::WebAPI::Plugin::Project; use Mojo::Base 'Mojolicious::Plugin'; -use Data::Dumper; + +use Mojo::JSON qw(decode_json encode_json); my $initialized = 0; my @projects; @@ -28,6 +29,7 @@ my $last_init_warning; my $caching = 1; my $cache_dir = '.cache'; +my $cache_filename = 'project'; sub register { my ($self, $app) = @_; @@ -47,30 +49,61 @@ sub register { sub _init_if_needed { return 1 if $initialized; my ($c) = @_; + my $wasdberror = 1; + my $err = 'Unknown'; eval { #the table may be missing - no big deal (only reports might be inaccurate if some other error occurred). - @projects = $c->schema->resultset('Project')->search(undef, { order_by => { -desc => [qw/prio name/] } }); - $initialized = 1; + eval { + my @rows = $c->schema->resultset('Project')->search(undef, { order_by => { -desc => [qw/prio name/] } }); + # we want to cache it, so move to simpler structure + for my $r (@rows) { + my %proj = ( id => $r->id, name => $r->name, path => $r->path, redirect => $r->redirect ); + push @projects, \%proj; + } + $wasdberror = 0; + $initialized = 1; + }; + $err = $@; + if ($wasdberror && !@projects) { + my $f = Mojo::File->new( "$cache_dir/$cache_filename.json" ); + if (-r $f) { + eval { + my $body = $f->slurp; + my $json = decode_json($body); + @projects = @$json; + $c->log->error($c->dumper("Loaded projects from cache", \@projects)); + $initialized = 1; + }; + } + } 1; - } or do { + }; + if ($wasdberror) { if (!$last_init_warning || 600 < time() - $last_init_warning) { - $c->log->error(Dumper("Cannot load projects", $@)); + $c->log->error($c->dumper("Cannot load projects", $err)); $last_init_warning = time(); } - return 0; + return 0 unless $initialized; }; + if ($caching && @projects && !$wasdberror) { + eval { + my $f = Mojo::File->new( "$cache_dir/$cache_filename.json" ); + $f->spurt(encode_json([ @projects ])); + }; + } + for my $p (@projects) { - my $name = $p->name; - my $id = $p->id; + my $name = $p->{name}; + my $id = $p->{id}; my $alias = $name; $alias =~ tr/ //ds; # remove spaces $alias =~ tr/\.//ds; # remove dots $alias = "c$alias" if $alias =~ /^\d/; $alias = lc($alias); - $projects_path{$name} = $p->path; - $projects_path{$id} = $p->path; + $projects_path{$name} = $p->{path}; + $projects_path{$id} = $p->{path}; $projects_alias{$name} = $alias; - my $redirect = $p->redirect; + my $redirect = $p->{redirect}; next unless $redirect; my @parts = split ';', $redirect; for my $r (@parts) { @@ -91,7 +124,7 @@ sub _list { my @res; for my $p (@projects) { - push @res, $p->name; + push @res, $p->{name}; } return \@res; } @@ -102,7 +135,7 @@ sub _list_full { my @res; for my $p (@projects) { - my $name = $p->name; + my $name = $p->{name}; my $alias = $projects_alias{$name}; my $path = $projects_path{$name}; @@ -117,7 +150,7 @@ sub _redirect { _init_if_needed($c); for my $p (@projects) { - my $name = $p->name; + my $name = $p->{name}; my $redirect; $redirect = $projects_region_redirect{$region}{$name} if $region; $redirect = $projects_redirect{$name} unless $redirect; @@ -133,7 +166,7 @@ sub _get_id { _init_if_needed($c); for my $p (@projects) { - my $id = $p->id; + my $id = $p->{id}; my $ppath = $projects_path{$id}; return $id if (0 == rindex($path, $ppath, 0)); } diff --git a/t/environ/01-smoke-offline-remote.sh b/t/environ/01-smoke-offline-remote.sh index 96d468be..f7258bc1 100755 --- a/t/environ/01-smoke-offline-remote.sh +++ b/t/environ/01-smoke-offline-remote.sh @@ -51,7 +51,7 @@ $mc/curl /download/folder1/file1.1.dat.metalink | grep -C 10 "http://$($ap6/prin $mc/curl /download/folder1/file1.1.dat.meta4 | grep -C 10 "http://$($ap6/print_address)/folder1/file1.1.dat<" | grep "$($ap5/print_address)" echo try restart the service and check the same -$mc/stop && ENVIRON_MC_DB_AUSTOSTART=0 $mc/start +$mc/stop && ENVIRON_MC_DB_AUTOSTART=0 $mc/start $mc/curl -I /download/folder1/file1.1.dat | grep -E "$($ap6/print_address)|$($ap5/print_address)"/folder1/file1.1.dat $mc/curl /download/folder1/file1.1.dat.metalink | grep -C 10 "http://$($ap6/print_address)/folder1/file1.1.dat<" | grep "$($ap5/print_address)" diff --git a/t/environ/01-smoke-offline.sh b/t/environ/01-smoke-offline.sh index f7fc3413..cb70ff2f 100755 --- a/t/environ/01-smoke-offline.sh +++ b/t/environ/01-smoke-offline.sh @@ -46,7 +46,7 @@ $mc/curl /download/folder1/file1.1.dat.meta4 | grep -C 10 "http://$($ap6/prin $mc/curl /download/folder1/ | grep file1.1.dat echo try restart the service and check the same -$mc/stop && ENVIRON_MC_DB_AUSTOSTART=0 $mc/start +$mc/stop && ENVIRON_MC_DB_AUTOSTART=0 $mc/start $mc/curl -I /download/folder1/file1.1.dat | grep -E "$($ap6/print_address)|$($ap5/print_address)"/folder1/file1.1.dat $mc/curl /download/folder1/file1.1.dat.metalink | grep -C 10 "http://$($ap6/print_address)/folder1/file1.1.dat<" | grep "$($ap5/print_address)" diff --git a/t/environ/14-project.sh b/t/environ/14-project.sh index 06b5aaf9..2ede5267 100755 --- a/t/environ/14-project.sh +++ b/t/environ/14-project.sh @@ -45,9 +45,9 @@ $mc/backstage/job -e mirror_scan -a '["/project1/folder1"]' $mc/backstage/shoot -$mc/curl -s /rest/project/proj1 -$mc/curl -s /rest/project/proj1/mirror_summary -$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?4' | grep -E '"outdated":"?0' +$mc/curl /rest/project/proj1 +$mc/curl /rest/project/proj1/mirror_summary +$mc/curl /rest/project/proj1/mirror_summary | grep -E '"current":"?4' | grep -E '"outdated":"?0' $mc/backstage/job -e folder_sync -a '["/project1/folder2"]' $mc/backstage/job -e mirror_scan -a '["/project1/folder2"]' @@ -61,21 +61,34 @@ $mc/backstage/job -e report -a '["once"]' $mc/backstage/shoot rc=0 -$mc/curl -s /rest/repmirror | grep -F '"country":"jp","proj1score":"0","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"'$($ap4/print_address)'"' || rc=$? +$mc/curl /rest/repmirror | grep -F '"country":"jp","proj1score":"0","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"'$($ap4/print_address)'"' || rc=$? echo proj1 is not on ap4, so it shouldnt appear in repmirror at all test $rc -gt 0 -$mc/curl -s /rest/repmirror | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}' +$mc/curl /rest/repmirror | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}' + +$mc/curl /rest/project | grep -F '{"alias":"proj2","name":"proj 2","path":"\/project2"}' | grep -F '{"alias":"proj1","name":"proj1","path":"\/project1"}' echo ceck the same when DB is offline $mc/db/stop -$mc/curl -s /rest/repmirror | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}' +$mc/curl /rest/repmirror | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}' + +$mc/curl /rest/project | grep -F '{"alias":"proj2","name":"proj 2","path":"\/project2"}' | grep -F '{"alias":"proj1","name":"proj1","path":"\/project1"}' + +echo now restart the service while DB is offline +$mc/stop +ENVIRON_MC_DB_AUTOSTART=0 $mc/start + +$mc/curl /rest/repmirror | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}' + +$mc/curl /rest/project | grep -F '{"alias":"proj2","name":"proj 2","path":"\/project2"}' | grep -F '{"alias":"proj1","name":"proj1","path":"\/project1"}' + $mc/db/start -$mc/curl -s /rest/project/proj1/mirror_summary -$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?2' | grep -E '"outdated":"?2' +$mc/curl /rest/project/proj1/mirror_summary +$mc/curl /rest/project/proj1/mirror_summary | grep -E '"current":"?2' | grep -E '"outdated":"?2' -$mc/curl -s /rest/project/proj1/mirror_list | grep -E '{"current":"?1"?,"server_id":"?1"?,"url":"127.0.0.1:1294"},{"current":"?1"?,"server_id":"?3"?,"url":"127.0.0.1:1314"},{"current":"?0"?,"server_id":"?4"?,"url":"127.0.0.1:1284"},{"current":"?0"?,"server_id":"?2"?,"url":"127.0.0.1:1304"}' +$mc/curl /rest/project/proj1/mirror_list | grep -E '{"current":"?1"?,"server_id":"?1"?,"url":"127.0.0.1:1294"},{"current":"?1"?,"server_id":"?3"?,"url":"127.0.0.1:1314"},{"current":"?0"?,"server_id":"?4"?,"url":"127.0.0.1:1284"},{"current":"?0"?,"server_id":"?2"?,"url":"127.0.0.1:1304"}' echo request from jp should be redirected to ap4 $mc/curl -I /download/project2/folder1/file1.1.dat?COUNTRY=jp | grep $($ap4/print_address) diff --git a/t/lib/environ/mc/source/_exec.m4 b/t/lib/environ/mc/source/_exec.m4 index 655d80ed..dd536098 100644 --- a/t/lib/environ/mc/source/_exec.m4 +++ b/t/lib/environ/mc/source/_exec.m4 @@ -3,7 +3,7 @@ __workdir/gen_env set -a source __workdir/conf.env set +a -test 1 != "${ENVIRON_MC_DB_AUSTOSTART-1}" || __workdir/db/status >& /dev/null || __workdir/db/start "--transaction-isolation=read-committed" +test 1 != "${ENVIRON_MC_DB_AUTOSTART-1}" || __workdir/db/status >& /dev/null || __workdir/db/start "--transaction-isolation=read-committed" [ -e __workdir/db/sql_mc_test ] || __workdir/db/create_db mc_test (