Cache projects from DB

openSUSE · Jun 15, 2023 · dcff388 · dcff388
1 parent ef05456
commit dcff388
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 30 deletions.
diff --git a/lib/MirrorCache/WebAPI/Controller/Rest/ReportMirror.pm b/lib/MirrorCache/WebAPI/Controller/Rest/ReportMirror.pm
@@ -91,7 +91,7 @@ sub list {
     return unless $self->mcproject->caching;
 
     if (!$last_cache_run || 60 < time() - $last_cache_run) {
-        my $wassaveerror = 1;
+        $last_cache_run = time();
         eval {
             my $f = Mojo::File->new( $self->mcproject->cache_dir . "/$cache_filename.json" );
             $f->spurt($body) if $body;
@@ -102,9 +102,8 @@ sub list {
                 $f = Mojo::File->new( $self->mcproject->cache_dir . "/$cache_filename$region.json" );
                 $f->spurt($x);
             }
-            $wassaveerror = 0;
             1;
-        } or $last_cache_run = time();
+        }
     }
 }
 

diff --git a/lib/MirrorCache/WebAPI/Plugin/Project.pm b/lib/MirrorCache/WebAPI/Plugin/Project.pm
@@ -16,7 +16,8 @@
 
 package MirrorCache::WebAPI::Plugin::Project;
 use Mojo::Base 'Mojolicious::Plugin';
-use Data::Dumper;
+
+use Mojo::JSON qw(decode_json encode_json);
 
 my $initialized = 0;
 my @projects;
@@ -28,6 +29,7 @@ my $last_init_warning;
 
 my $caching   = 1;
 my $cache_dir = '.cache';
+my $cache_filename = 'project';
 
 sub register {
     my ($self, $app) = @_;
@@ -47,30 +49,61 @@ sub register {
 sub _init_if_needed {
     return 1 if $initialized;
     my ($c) = @_;
+    my $wasdberror = 1;
+    my $err = 'Unknown';
     eval { #the table may be missing - no big deal (only reports might be inaccurate if some other error occurred).
-        @projects = $c->schema->resultset('Project')->search(undef, { order_by => { -desc => [qw/prio name/] } });
-        $initialized = 1;
+        eval {
+            my @rows = $c->schema->resultset('Project')->search(undef, { order_by => { -desc => [qw/prio name/] } });
+            # we want to cache it, so move to simpler structure
+            for my $r (@rows) {
+                my %proj = ( id => $r->id, name => $r->name, path => $r->path, redirect => $r->redirect );
+                push @projects, \%proj;
+            }
+            $wasdberror = 0;
+            $initialized = 1;
+        };
+        $err = $@;
+        if ($wasdberror && !@projects) {
+            my $f = Mojo::File->new( "$cache_dir/$cache_filename.json" );
+            if (-r $f) {
+                eval {
+                    my $body = $f->slurp;
+                    my $json = decode_json($body);
+                    @projects = @$json;
+                    $c->log->error($c->dumper("Loaded projects from cache", \@projects));
+                    $initialized = 1;
+                };
+            }
+        }
         1;
-    } or do {
+    };
+    if ($wasdberror) {
         if (!$last_init_warning || 600 < time() - $last_init_warning) {
-            $c->log->error(Dumper("Cannot load projects", $@));
+            $c->log->error($c->dumper("Cannot load projects", $err));
             $last_init_warning = time();
         }
-        return 0;
+        return 0 unless $initialized;
     };
 
+    if ($caching && @projects && !$wasdberror) {
+        eval {
+            my $f = Mojo::File->new( "$cache_dir/$cache_filename.json" );
+            $f->spurt(encode_json([ @projects ]));
+        };
+    }
+
     for my $p (@projects) {
-        my $name  = $p->name;
-        my $id    = $p->id;
+        my $name  = $p->{name};
+        my $id    = $p->{id};
         my $alias = $name;
         $alias =~ tr/ //ds;  # remove spaces
         $alias =~ tr/\.//ds; # remove dots
         $alias = "c$alias" if $alias =~ /^\d/;
         $alias = lc($alias);
-        $projects_path{$name} = $p->path;
-        $projects_path{$id}   = $p->path;
+        $projects_path{$name} = $p->{path};
+        $projects_path{$id}   = $p->{path};
         $projects_alias{$name} = $alias;
-        my $redirect = $p->redirect;
+        my $redirect = $p->{redirect};
         next unless $redirect;
         my @parts = split ';', $redirect;
         for my $r (@parts) {
@@ -91,7 +124,7 @@ sub _list {
 
     my @res;
     for my $p (@projects) {
-        push @res, $p->name;
+        push @res, $p->{name};
     }
     return \@res;
 }
@@ -102,7 +135,7 @@ sub _list_full {
 
     my @res;
     for my $p (@projects) {
-        my $name  = $p->name;
+        my $name  = $p->{name};
         my $alias = $projects_alias{$name};
         my $path  = $projects_path{$name};
 
@@ -117,7 +150,7 @@ sub _redirect {
     _init_if_needed($c);
 
     for my $p (@projects) {
-        my $name  = $p->name;
+        my $name  = $p->{name};
         my $redirect;
         $redirect = $projects_region_redirect{$region}{$name} if $region;
         $redirect = $projects_redirect{$name} unless $redirect;
@@ -133,7 +166,7 @@ sub _get_id {
     _init_if_needed($c);
 
     for my $p (@projects) {
-        my $id = $p->id;
+        my $id = $p->{id};
         my $ppath  = $projects_path{$id};
         return $id if (0 == rindex($path, $ppath, 0));
     }

diff --git a/t/environ/01-smoke-offline-remote.sh b/t/environ/01-smoke-offline-remote.sh
@@ -51,7 +51,7 @@ $mc/curl /download/folder1/file1.1.dat.metalink | grep -C 10 "http://$($ap6/prin
 $mc/curl /download/folder1/file1.1.dat.meta4    | grep -C 10 "http://$($ap6/print_address)/folder1/file1.1.dat<" | grep "$($ap5/print_address)"
 
 echo try restart the service and check the same
-$mc/stop && ENVIRON_MC_DB_AUSTOSTART=0 $mc/start
+$mc/stop && ENVIRON_MC_DB_AUTOSTART=0 $mc/start
 
 $mc/curl -I /download/folder1/file1.1.dat       | grep -E "$($ap6/print_address)|$($ap5/print_address)"/folder1/file1.1.dat
 $mc/curl /download/folder1/file1.1.dat.metalink | grep -C 10 "http://$($ap6/print_address)/folder1/file1.1.dat<" | grep "$($ap5/print_address)"

diff --git a/t/environ/01-smoke-offline.sh b/t/environ/01-smoke-offline.sh
@@ -46,7 +46,7 @@ $mc/curl /download/folder1/file1.1.dat.meta4    | grep -C 10 "http://$($ap6/prin
 $mc/curl /download/folder1/ | grep file1.1.dat
 
 echo try restart the service and check the same
-$mc/stop && ENVIRON_MC_DB_AUSTOSTART=0 $mc/start
+$mc/stop && ENVIRON_MC_DB_AUTOSTART=0 $mc/start
 
 $mc/curl -I /download/folder1/file1.1.dat          | grep -E "$($ap6/print_address)|$($ap5/print_address)"/folder1/file1.1.dat
 $mc/curl /download/folder1/file1.1.dat.metalink | grep -C 10 "http://$($ap6/print_address)/folder1/file1.1.dat<" | grep "$($ap5/print_address)"

diff --git a/t/environ/14-project.sh b/t/environ/14-project.sh
@@ -45,9 +45,9 @@ $mc/backstage/job -e mirror_scan -a '["/project1/folder1"]'
 $mc/backstage/shoot
 
 
-$mc/curl -s /rest/project/proj1
-$mc/curl -s /rest/project/proj1/mirror_summary
-$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?4' | grep -E '"outdated":"?0'
+$mc/curl /rest/project/proj1
+$mc/curl /rest/project/proj1/mirror_summary
+$mc/curl /rest/project/proj1/mirror_summary | grep -E '"current":"?4' | grep -E '"outdated":"?0'
 
 $mc/backstage/job -e folder_sync -a '["/project1/folder2"]'
 $mc/backstage/job -e mirror_scan -a '["/project1/folder2"]'
@@ -61,21 +61,34 @@ $mc/backstage/job -e report -a '["once"]'
 $mc/backstage/shoot
 
 rc=0
-$mc/curl -s /rest/repmirror  | grep -F '"country":"jp","proj1score":"0","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"'$($ap4/print_address)'"' || rc=$?
+$mc/curl /rest/repmirror  | grep -F '"country":"jp","proj1score":"0","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"'$($ap4/print_address)'"' || rc=$?
 echo proj1 is not on ap4, so it shouldnt appear in repmirror at all
 test $rc -gt 0
 
-$mc/curl -s /rest/repmirror  | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}'
+$mc/curl /rest/repmirror  | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}'
+
+$mc/curl /rest/project | grep -F '{"alias":"proj2","name":"proj 2","path":"\/project2"}' | grep -F '{"alias":"proj1","name":"proj1","path":"\/project1"}'
 
 echo ceck the same when DB is offline
 $mc/db/stop
-$mc/curl -s /rest/repmirror  | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}'
+$mc/curl /rest/repmirror  | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}'
+
+$mc/curl /rest/project | grep -F '{"alias":"proj2","name":"proj 2","path":"\/project2"}' | grep -F '{"alias":"proj1","name":"proj1","path":"\/project1"}'
+
+echo now restart the service while DB is offline
+$mc/stop
+ENVIRON_MC_DB_AUTOSTART=0 $mc/start
+
+$mc/curl /rest/repmirror  | grep -F '{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}'
+
+$mc/curl /rest/project | grep -F '{"alias":"proj2","name":"proj 2","path":"\/project2"}' | grep -F '{"alias":"proj1","name":"proj1","path":"\/project1"}'
+
 $mc/db/start
 
-$mc/curl -s /rest/project/proj1/mirror_summary
-$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?2' | grep -E '"outdated":"?2'
+$mc/curl /rest/project/proj1/mirror_summary
+$mc/curl /rest/project/proj1/mirror_summary | grep -E '"current":"?2' | grep -E '"outdated":"?2'
 
-$mc/curl -s /rest/project/proj1/mirror_list | grep -E '{"current":"?1"?,"server_id":"?1"?,"url":"127.0.0.1:1294"},{"current":"?1"?,"server_id":"?3"?,"url":"127.0.0.1:1314"},{"current":"?0"?,"server_id":"?4"?,"url":"127.0.0.1:1284"},{"current":"?0"?,"server_id":"?2"?,"url":"127.0.0.1:1304"}'
+$mc/curl /rest/project/proj1/mirror_list | grep -E '{"current":"?1"?,"server_id":"?1"?,"url":"127.0.0.1:1294"},{"current":"?1"?,"server_id":"?3"?,"url":"127.0.0.1:1314"},{"current":"?0"?,"server_id":"?4"?,"url":"127.0.0.1:1284"},{"current":"?0"?,"server_id":"?2"?,"url":"127.0.0.1:1304"}'
 
 echo request from jp should be redirected to ap4
 $mc/curl -I /download/project2/folder1/file1.1.dat?COUNTRY=jp | grep $($ap4/print_address)

diff --git a/t/lib/environ/mc/source/_exec.m4 b/t/lib/environ/mc/source/_exec.m4
@@ -3,7 +3,7 @@ __workdir/gen_env
 set -a
 source __workdir/conf.env
 set +a
-test 1 != "${ENVIRON_MC_DB_AUSTOSTART-1}" || __workdir/db/status >& /dev/null || __workdir/db/start "--transaction-isolation=read-committed"
+test 1 != "${ENVIRON_MC_DB_AUTOSTART-1}" || __workdir/db/status >& /dev/null || __workdir/db/start "--transaction-isolation=read-committed"
 [ -e __workdir/db/sql_mc_test ] || __workdir/db/create_db mc_test
 
 (