Merge pull request #819 from mgcam/warn_about_shadows

Added a log message about deletable shadow folders.
wtsi-npg · Feb 7, 2024 · e1f8f79 · e1f8f79
2 parents f3f7697 + c748362
commit e1f8f79
Show file tree

Hide file tree

Showing 2 changed files with 100 additions and 4 deletions.
diff --git a/lib/npg_pipeline/validation.pm b/lib/npg_pipeline/validation.pm
@@ -28,7 +28,9 @@ with qw{npg_pipeline::validation::common
 our $VERSION = '0';
 
 Readonly::Array  my @NPG_DELETABLE_UNCOND => ('run cancelled', 'data discarded');
-Readonly::Array  my @NPG_DELETABLE_STATES => (@NPG_DELETABLE_UNCOND,'qc complete');
+Readonly::Scalar my $QC_COMPLETE_STATUS   => q[qc complete];
+Readonly::Array  my @NPG_DELETABLE_STATES =>
+                      (@NPG_DELETABLE_UNCOND, $QC_COMPLETE_STATUS);
 Readonly::Scalar my $MIN_KEEP_DAYS        => 14;
 Readonly::Scalar my $UNREALISTIC_NUM_STAGING_PP_FILES => 10_000;
 Readonly::Scalar my $DEFAULT_IRODS_ROOT   => q[/seq];
@@ -445,6 +447,16 @@ sub run {
 
   $self->_flagged_as_not_deletable() and return 0;
 
+  # For now just inform about a deletable shadow run folder. The folder
+  # will be deleted manually.
+  # Later, when we gain confidence, we might return 1 here, which will
+  # result in the automatic deletion.
+  if ($self->_is_deletable_shadow_runfolder()) {
+    $self->info(sprintf 'Shadow runfolder %s for run %i can be deleted',
+      $self->runfolder_path, $self->id_run);
+    return 0;
+  }
+
   my $deletable = $self->_npg_tracking_deletable('unconditional');
   my $vars_set  = 0;
 
@@ -956,6 +968,16 @@ sub _file_archive_deletable {
   return $deletable;
 }
 
+sub _is_deletable_shadow_runfolder {
+  my $self = shift;
+  if ( ($self->runfolder_path() =~ m{/incoming/}xms) &&
+       ($self->_run_status_obj->description eq $QC_COMPLETE_STATUS) &&
+       !$self->tracking_run->is_tag_set($STAGING_TAG)) {
+    return 1;
+  }
+  return 0;
+}
+
 ###########################################################
 # The code of this function was copied from
 # L<https://github.com/wtsi-npg/npg_irods/blob/63d3485c44cc00e30d8a8ec5bcdc23d2297f0d39/bin/npg_publish_tree.pl#L81> 

diff --git a/t/10-validation.t b/t/10-validation.t
@@ -1,11 +1,11 @@
 use strict;
 use warnings;
-use Test::More tests => 8;
+use Test::More tests => 9;
 use Test::Exception;
 use Test::Warn;
 use Test::Trap qw/ :warn /;
 use File::Path qw/ make_path /;
-use File::Slurp qw/ write_file /;
+use File::Slurp qw/ read_file write_file /;
 use File::Copy;
 use Log::Log4perl qw/ :levels /;
 use File::Temp qw/ tempdir /;
@@ -19,7 +19,7 @@ use_ok ('npg_pipeline::validation');
 my $util = t::util->new();
 my $logfile = join q[/], $util->temp_directory(), 'logfile';
 Log::Log4perl->easy_init({layout => '%d %-5p %c - %m%n',
-                          level  => $WARN,
+                          level  => $INFO,
                           file   => $logfile,
                           utf8   => 1});
 
@@ -356,4 +356,78 @@ subtest 'presence of onboard analysis results' => sub {
     'NovaSeq - no onboard analysis - deletable');
 };
 
+subtest 'shadow runfolder' => sub {
+  plan tests => 7;
+
+  my $id_run = 35348;
+  my $run_row = $tracking_schema->resultset('Run')->find($id_run);
+  $run_row or die "Run $id_run is missing from the test tracking database";
+
+  my $staging_path = tempdir(CLEANUP => 1);
+  $run_row->update({folder_path_glob => "$staging_path/*/"});
+
+  my $runfolder_path = join q[/], $staging_path, 'incoming',
+                                  $run_row->folder_name;
+  make_path($runfolder_path);
+  copy('t/data/novaseq/35843_RunInfo.xml',"$runfolder_path/RunInfo.xml");
+  copy('t/data/novaseq/35843_RunParameters.xml',"$runfolder_path/RunParameters.xml");
+
+  my $qc_status = 'archival pending';
+  my $status_disc_row = $tracking_schema->resultset('RunStatusDict')
+    ->search({description => $qc_status})->next();
+  my $rs = $tracking_schema->resultset('RunStatus');
+  map { $_->update({iscurrent => 0}) } $rs->search({id_run => $id_run})->all();
+  $rs->create({
+    id_run_status_dict => $status_disc_row->id_run_status_dict,
+    id_run => $id_run,
+    id_user => 1,
+    iscurrent => 1
+  });
+  ok ($run_row->current_run_status_description() eq $qc_status,
+    "current status is '$qc_status'");
+  $run_row->set_tag('joe_admin','staging'); 
+
+  my $v = npg_pipeline::validation->new(
+    qc_schema           => $qc_schema,
+    npg_tracking_schema => $tracking_schema,
+    id_run              => $id_run,
+    runfolder_path      => $runfolder_path
+  );
+  ok (!$v->_is_deletable_shadow_runfolder(), 'shadow run folder is not deletable');
+
+  $qc_status = 'qc complete';
+  $status_disc_row = $tracking_schema->resultset('RunStatusDict')
+    ->search({description => $qc_status})->next();
+  map { $_->update({iscurrent => 0}) } $rs->search({id_run => $id_run})->all();
+  $rs->create({
+    id_run_status_dict => $status_disc_row->id_run_status_dict,
+    id_run => $id_run,
+    id_user => 1,
+    iscurrent => 1
+  });
+  ok ($run_row->current_run_status_description() eq $qc_status,
+    "current status is '$qc_status'");
+
+  $v = npg_pipeline::validation->new(
+    qc_schema           => $qc_schema,
+    npg_tracking_schema => $tracking_schema,
+    id_run              => $id_run,
+    runfolder_path      => $runfolder_path
+  );
+  ok (!$v->_is_deletable_shadow_runfolder(), 'shadow run folder is not deletable');
+
+  $run_row->unset_tag('staging');
+  $v = npg_pipeline::validation->new(
+    qc_schema           => $qc_schema,
+    npg_tracking_schema => $tracking_schema,
+    id_run              => $id_run,
+    runfolder_path      => $runfolder_path
+  );
+  ok ($v->_is_deletable_shadow_runfolder(), 'shadow run folder is deletable');
+  ok (!$v->run(), q[..., but the return value is 'not deletable']);
+  like (read_file($logfile),
+    qr/Shadow runfolder $runfolder_path for run $id_run can be deleted/,
+    'expected info in the log file');
+};
+
 1;