Rework mirrors report generation

* use tables server_project and rollout instead of column project.etalon and table folder_diff_server
* drop column project.etalon
* remove api routest mirror_list and mirror_summary in /rest/project/:name/

assets/javascripts/reporttable.js

@@ -14,7 +14,6 @@ function setupReportTable() {
             columnName = 'c' + columnName;
         }
         columns.push({ data: (columnName + 'score'), defaultContent: "" });
-        columns.push({ data: (columnName + 'victim'), defaultContent: "" });
     });
 
     var url = $("#reporttable_api_url").val();
@@ -44,10 +43,6 @@ function setupReportTable() {
 function layoutReportTable() {
     var dt = $('.reporttable').DataTable();
 
-    var victim = 1;
-    if (!$('#victimcheckbox').is(":checked")) {
-        victim = 0;
-    }
     var existchecked = 0;
     $('#checkboxes label').each(function() {
         var columnName = $(this).text();
@@ -80,9 +75,7 @@ function layoutReportTable() {
         if (vis != dt.columns(index).visible()) {
             dt.columns(index).visible(vis);
         }
-        if (vis && victim != dt.columns(index + 1).visible()) {
-            dt.columns(index + 1).visible(victim);
-        } else if (!vis && dt.columns(index + 1).visible()) {
+        if (!vis && dt.columns(index + 1).visible()) {
             dt.columns(index + 1).visible(vis);
         }
         i = i + 1;

lib/MirrorCache/Schema/ResultSet/Project.pm

@@ -8,60 +8,4 @@ use base 'DBIx::Class::ResultSet';
 use Mojo::File qw(path);
 
 
-sub mirror_summary {
-    my ($self, $name) = @_;
-    my $dbh = $self->result_source->schema->storage->dbh;
-
-    my $sql = <<"END_SQL";
-    select sum(case when diff = 0 then 1 else 0 end) as current, sum(case when diff = 0 or diff is null then 0 else 1 end) as outdated
-from (
-select
-    server_id,
-    max(case when d1 <> d2 or d1 is null then 1 else 0 end) diff
-from (
-select s.id as server_id, prj.name, fds.folder_diff_id d1, fds2.folder_diff_id d2
-from server s
-join project prj on prj.name = ?
-join folder f on f.path like concat(prj.path,'/%')
-join folder_diff fd on fd.folder_id = f.id
-left join folder_diff_server fds  on fds.folder_diff_id = fd.id and fds.server_id = s.id
-join folder_diff_server fds2 on fds2.folder_diff_id = fd.id and fds2.server_id = prj.etalon
-) x
-group by server_id
-having  sum(case when d1 is not null then 1 else 0 end) > 0
-) xx
-END_SQL
-    my $prep = $dbh->prepare($sql);
-    $prep->execute($name);
-    return $dbh->selectrow_hashref($prep);
-}
-
-sub mirror_list {
-    my ($self, $name) = @_;
-    my $dbh = $self->result_source->schema->storage->dbh;
-
-    my $sql = <<"END_SQL";
-select
-    s.id server_id,
-    min(case when d1 = d2 then 1 else 0 end) as current,
-    concat(s.hostname, s.urldir) as url
-from (
-select s.id as server_id, prj.name, fds.folder_diff_id d1, fds2.folder_diff_id d2
-from server s
-join project prj on prj.name = ?
-join folder f on f.path like concat(prj.path,'/%')
-join folder_diff fd on fd.folder_id = f.id
-left join folder_diff_server fds  on fds.folder_diff_id = fd.id and fds.server_id = s.id
-join folder_diff_server fds2 on fds2.folder_diff_id = fd.id and fds2.server_id = prj.etalon
-) x
-join server s on x.server_id = s.id
-group by s.id, s.hostname, s.urldir
-having sum(case when d1 is not null then 1 else 0 end) > 0
-order by current desc, url
-END_SQL
-    my $prep = $dbh->prepare($sql);
-    $prep->execute($name);
-    return $dbh->selectall_arrayref($prep, { Slice => {} });
-}
-
 1;

lib/MirrorCache/Schema/ResultSet/Server.pm

@@ -307,58 +307,46 @@ sub report_mirrors {
 
     my $sql = <<"END_SQL";
 with
-etalon as (
-    select prj.id project_id, prj.name, f.id as folder_id, fd2.id as diff_id, f.path
-    from project prj
-    join folder f on f.path like concat(prj.path,'%')
-    join folder_diff fd2 on fd2.folder_id = f.id
-    join folder_diff_server fds2 on fd2.id = fds2.folder_diff_id and fds2.server_id = prj.etalon
-    -- where prj.name = 'repositories'
-    $where1
-),
-project_folder_count as (
-    select project_id, count(*) cnt
-    from etalon
-    group by project_id
+recent_rollouts as (
+select r1.project_id, r1.prefix, max(r1.id) last_rollout_id, max(r2.id) prelast_rollout_id
+from rollout r1
+left join rollout r2 on (r1.project_id, r1.prefix) = (r2.project_id, r2.prefix) and r2.id < r1.id
+group by r1.project_id, r1.prefix
 )
-select s.id, s.region, s.country,
+select s.id as server_id, name, prefix, name as project, country, region,
     s.sponsor, s.sponsor_url,
     s.hostname as hostname,
     concat(s.hostname, s.urldir) as url,
     case when (select rating from server_stability where capability = 'http'  and server_id = s.id) > 0 then concat('http://',  s.hostname, '/', s.urldir, '/') else '' end as http_url,
     case when (select rating from server_stability where capability = 'https' and server_id = s.id) > 0 then concat('https://', s.hostname, '/', s.urldir, '/') else '' end as https_url,
     ( select msg from server_note where kind = 'Ftp'   and server_note.hostname = s.hostname order by server_note.dt desc limit 1) as ftp_url,
     ( select msg from server_note where kind = 'Rsync' and server_note.hostname = s.hostname order by server_note.dt desc limit 1) as rsync_url,
-    project,
-    round(case when project_folder_count.cnt > 3 then s_eq * 100 / project_folder_count.cnt when s_eq =  project_folder_count.cnt then 100 else 50 end, 0) score,
-    s_eq, s_ne, victim, project_folder_count.cnt
-from (
-select
-    cmp.project_id,
-    cmp.name project,
-    cmp.server_id,
-    sum(s_eq) s_eq, sum(s_ne) s_ne,
-    max(example) victim
-from (
-    select
-        etalon.project_id,
-        etalon.name,
-        case when etalon.diff_id = fd.id then 1 else 0 end as s_eq,
-        case when etalon.diff_id != fd.id or fd.id is null then 1 else 0 end as s_ne,
-        case when etalon.diff_id != fd.id or fd.id is null then path else '' end example,
-        fds.server_id
-    from etalon
-    left join folder_diff fd on fd.folder_id = etalon.folder_id
-    left join folder_diff_server fds on fds.folder_diff_id = fd.id
-) cmp
-group by server_id, project_id, name
-) smry
-join project_folder_count on project_folder_count.project_id = smry.project_id
-join server s on smry.server_id = s.id and s.enabled
-order by region, country, score, hostname, project;
+    case when recent_rollout_server.server_id is null then
+             case when sp.state > 0 then 100 else 0 end
+         when  6*60*60 > unix_timestamp(last_dt)    - unix_timestamp(rr2_dt) then 100
+         when 24*60*60 > unix_timestamp(prelast_dt) - unix_timestamp(rr1_dt) then 75
+         else 50
+    end as score
+from
+server s
+left join server_project sp on s.id = sp.server_id
+left join project on sp.project_id = project.id
+left join (
+select rs1.server_id, project_id, prefix, rs1.dt prelast_dt, rs2.dt last_dt, (select dt from rollout where id = rs1.rollout_id) rr1_dt, (select dt from rollout where id = rs2.rollout_id) rr2_dt
+from recent_rollouts
+left join rollout_server rs1 on rs1.rollout_id = recent_rollouts.prelast_rollout_id -- prelast
+left join rollout_server rs2 on rs2.rollout_id = recent_rollouts.last_rollout_id and rs1.server_id = rs2.server_id -- last
+) recent_rollout_server on recent_rollout_server.server_id = s.id and recent_rollout_server.project_id = project.id
+where
+s.enabled
+and coalesce(sp.state,1) > 0
+order by region, country, hostname
 END_SQL
 
-    $sql =~ s/interval '1 day'/interval 1 day/g unless ($dbh->{Driver}->{Name} eq 'Pg');
+    if ($dbh->{Driver}->{Name} eq 'Pg') {
+        $sql =~ s/unix_timestamp\(last_dt\)    - unix_timestamp\(rr2_dt\)/cast(EXTRACT(EPOCH FROM    last_dt) - EXTRACT(EPOCH FROM rr2_dt) as integer)/g;
+        $sql =~ s/unix_timestamp\(prelast_dt\) - unix_timestamp\(rr1_dt\)/cast(EXTRACT(EPOCH FROM prelast_dt) - EXTRACT(EPOCH FROM rr1_dt) as integer)/g;
+    }
 
     my $prep = $dbh->prepare($sql);
     if ($project && $region) {

lib/MirrorCache/Task/Report.pm

@@ -150,9 +150,10 @@ sub _run_mirrors {
     my %capability;
     my %url;
     for my $m (@$mirrors) {
+        next unless $m->{region} && $m->{country} && $m->{hostname} && $m->{project};
         $report{ $m->{region} }{ $m->{country} }{ $m->{hostname} }{ $m->{project} }
-          = [ $m->{score}, $m->{victim} ];
-        $sponsor{$m->{hostname}} = [$m->{sponsor},$m->{sponsor_url}];
+          = $m->{score};
+        $sponsor{$m->{hostname}} = [$m->{sponsor},$m->{sponsor_url}] if $m->{sponsor};
         $url{$m->{hostname}} = $m->{url};
         for my $capability (qw/http https ipv4 ipv6 ftp rsync/) {
             $capability{$m->{hostname}}{$capability} = $m->{$capability . '_url'} if $m->{$capability . '_url'};
@@ -179,15 +180,12 @@ sub _run_mirrors {
 
                 my $by_project = $by_country->{$hostname};
                 for my $project (sort keys %$by_project) {
-                    my $p = $by_project->{$project};
-                    my $score = $p->[0];
-                    my $victim = $p->[1];
+                    my $score = $by_project->{$project};
                     $project =~ tr/ //ds;
                     $project =~ tr/\.//ds;
                     $project = lc($project);
                     $project = "c$project" if $project =~ /^\d/;
                     $row{$project . 'score'}  = $score;
-                    $row{$project . 'victim'} = $victim;
                 }
                 # add capabilities
                 for my $capability (qw/http https ipv4 ipv6 ftp rsync/) {

lib/MirrorCache/WebAPI.pm

@@ -181,8 +181,6 @@ sub _setup_webui {
     $rest_r->get('/project')->name('rest_project')->to('table#list', table => 'Project');
     # $rest_r->get('/project/:name')->to('project#show');
     $rest_r->get('/project/:id')->to('table#list', table => 'Project');
-    $rest_r->get('/project/:name/mirror_summary')->to('project#mirror_summary');
-    $rest_r->get('/project/:name/mirror_list')->to('project#mirror_list');
     $rest_r->get('/project/propagation/:project_id')->to('project_propagation#list');
     $rest_r->get('/rollout_server/:version')->to('rollout_server#list');
 

lib/MirrorCache/WebAPI/Controller/Rest/Project.pm

@@ -26,24 +26,6 @@ sub show {
     $self->render(json => {$prj->get_columns});
 }
 
-sub mirror_list {
-    my ($self) = @_;
-    my $name = $self->param("name");
-
-    my $rec = $self->schema->resultset('Project')->mirror_list($name);
-
-    $self->render(json => $rec);
-}
-
-sub mirror_summary {
-    my ($self) = @_;
-    my $name = $self->param("name");
-
-    my $rec = $self->schema->resultset('Project')->mirror_summary($name);
-
-    $self->render(json => { current => $rec->{current}, outdated => $rec->{outdated} });
-}
-
 sub list {
     my ($self) = @_;
 

lib/MirrorCache/resources/migrations/Pg.sql

@@ -412,4 +412,5 @@ alter table rollout_server add column if not exists scan_dt timestamp;
 create index if not exists rollout_version_inx on rollout(version);
 -- 35 up
 alter table folder_diff add column if not exists realfolder_id bigint;
-
+-- 36 up
+alter table project drop column etalon;

lib/MirrorCache/resources/migrations/mysql.sql

@@ -424,3 +424,5 @@ alter table rollout_server add column if not exists scan_dt timestamp;
 create index if not exists i_rollout_version on rollout(version);
 -- 35 up
 alter table folder_diff add column if not exists realfolder_id bigint;
+-- 36 up
+alter table project drop constraint fk_project_etalon, drop column etalon;

t/environ/14-project-capability.sh

@@ -27,7 +27,7 @@ $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($a
 $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"
 $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','us','na'"
 
-$mc/sql "insert into project(name,path,etalon) select 'proj1','/project1', 1"
+$mc/sql "insert into project(name,path) select 'proj1','/project1'"
 
 $mc/sql "insert into server_project(server_id,project_id,state) select 3,1,-1"
 

t/environ/14-project-hq-regions.sh

@@ -74,15 +74,16 @@ $mc8/sql_test 2 == "select count(*) from server"
 
 
 for i in 6 8 9; do
-    mc$i/sql "insert into project(name,path,etalon) select 'proj1','/project1', min(id) from server"
-    mc$i/sql "insert into project(name,path,etalon) select 'proj 2','/project2', min(id) from server"
+    mc$i/sql "insert into project(name,path) select 'proj1','/project1'"
+    mc$i/sql "insert into project(name,path) select 'proj 2','/project2'"
     mc$i/backstage/job -e folder_sync -a '["/project1/folder1"]'
     mc$i/backstage/job -e mirror_scan -a '["/project1/folder1"]'
     mc$i/backstage/job -e folder_sync -a '["/project1/folder2"]'
     mc$i/backstage/job -e mirror_scan -a '["/project1/folder2"]'
     mc$i/backstage/job -e folder_sync -a '["/project2/folder1"]'
     mc$i/backstage/job -e mirror_scan -a '["/project2/folder1"]'
     mc$i/backstage/shoot
+    mc$i/backstage/job mirror_probe_projects
     mc$i/backstage/job -e report -a '["once"]'
     mc$i/backstage/shoot
 done

t/environ/14-project-hq.sh

@@ -61,15 +61,16 @@ $mc8/sql "insert into server(hostname,urldir,enabled,country,region) select '$($
 $mc8/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','jp','as'"
 
 for i in 6 8 9; do
-    mc$i/sql "insert into project(name,path,etalon) select 'proj1','/project1', 1"
-    mc$i/sql "insert into project(name,path,etalon) select 'proj 2','/project2', 1"
+    mc$i/sql "insert into project(name,path) select 'proj1','/project1'"
+    mc$i/sql "insert into project(name,path) select 'proj 2','/project2'"
     mc$i/backstage/job -e folder_sync -a '["/project1/folder1"]'
     mc$i/backstage/job -e mirror_scan -a '["/project1/folder1"]'
     mc$i/backstage/job -e folder_sync -a '["/project1/folder2"]'
     mc$i/backstage/job -e mirror_scan -a '["/project1/folder2"]'
     mc$i/backstage/job -e folder_sync -a '["/project2/folder1"]'
     mc$i/backstage/job -e mirror_scan -a '["/project2/folder1"]'
     mc$i/backstage/shoot
+    mc$i/backstage/job mirror_probe_projects
     mc$i/backstage/job -e report -a '["once"]'
     mc$i/backstage/shoot
 done
@@ -121,6 +122,7 @@ test 8 == $($mc9/curl -i /report/mirrors | grep -A500 '200 OK' | grep -Eo $allmi
 echo collect report when one of the instances is down
 $mc6/stop
 
+$mc9/backstage/job mirror_probe_projects
 $mc9/backstage/job -e report -a '["once"]'
 $mc9/backstage/shoot
 

t/environ/14-project-report.sh

@@ -44,9 +44,9 @@ $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($a
 $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap4/print_address)','','t','jp','as'"
 $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap3/print_address)','','f','jp','as'"
 
-$mc/sql "insert into project(name,path,etalon) select '2.0 1','/project2/folder1', 3"
-$mc/sql "insert into project(name,path,etalon) select 'proj1','/project1', 3"
-$mc/sql "insert into project(name,path,etalon) select '2.0 2','/project2/folder2', 3"
+$mc/sql "insert into project(name,path) select '2.0 1','/project2/folder1'"
+$mc/sql "insert into project(name,path) select 'proj1','/project1'"
+$mc/sql "insert into project(name,path) select '2.0 2','/project2/folder2'"
 
 echo add extra info for the report
 $mc/sql "insert into server_note(dt,hostname,kind,msg) select now(), '$($ap7/print_address)','Ftp', 'ftp://ftp.ap7.com/opensuse'"
@@ -72,13 +72,12 @@ $mc/backstage/job -e folder_sync -a '["/project2/folder2"]'
 $mc/backstage/job -e mirror_scan -a '["/project2/folder2"]'
 $mc/backstage/shoot
 
+$mc/backstage/job mirror_probe_projects
 $mc/backstage/job -e report -a '["once"]'
 $mc/backstage/shoot
 
 $mc/curl /report/mirrors | tidy --drop-empty-elements no | \
-   grep -A5 -F '<div class="repo">' | \
-   grep -A4 -F '<a class="repouncertain"' | \
-   grep -A3 -F '"diff in: /project2/folder2"' | \
+   grep -A4 -F '<div class="repo">' | \
    grep -A2 -F '"http://127.0.0.1:1304/project2/folder2">' | \
    grep -C3 '\b2\b' | \
    grep -C3 -F '</a>'

t/environ/14-project-rollout-iso.sh

@@ -31,8 +31,8 @@ $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($a
 $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap5/print_address)','','t','cn','as'"
 $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap4/print_address)','','t','jp','as'"
 
-$mc/sql "insert into project(name,path,etalon) select 'proj1 ISO','/project1/iso', 3"
-$mc/sql "insert into project(name,path,etalon) select 'proj 2 ISO','/project2/iso', 3"
+$mc/sql "insert into project(name,path) select 'proj1 ISO','/project1/iso'"
+$mc/sql "insert into project(name,path) select 'proj 2 ISO','/project2/iso'"
 
 $mc/backstage/job -e folder_sync -a '["/project1/iso"]'
 $mc/backstage/job -e mirror_scan -a '["/project1/iso"]'

t/environ/14-project-rollout-repo-subfolder.sh

@@ -75,4 +75,10 @@ $mc/sql_test 4 == 'select count(*) from rollout'
 $mc/sql_test 1 == 'select count(*) from rollout_server where rollout_id = 4'
 $mc/sql_test 2 == 'select count(*) from rollout_server where rollout_id = 3'
 
+$mc/backstage/job mirror_probe_projects
+$mc/backstage/job report
+$mc/backstage/shoot
+
+$mc/curl /rest/repmirror
+
 echo success