Smart handle of local symlinks

Redirect to a mirror which has the file to which the symlink is pointing

lib/MirrorCache/Datamodule.pm

@@ -171,7 +171,7 @@ sub avoid_countries($self) {
 
 sub pedantic($self) {
     unless (defined $self->_pedantic) {
-        $self->_init_location;
+        $self->_init_path;
     }
     return $self->_pedantic;
 }
@@ -608,7 +608,7 @@ sub _init_path($self) {
         }
     }
 
-    $self->_pedantic($pedantic) if defined $pedantic;
+    $self->_pedantic($pedantic // 0);
 
     $self->agent; # parse headers
     if (

lib/MirrorCache/WebAPI/Plugin/Dir.pm

@@ -227,14 +227,17 @@ sub _render_dir {
 # this combines similar checks for redirecting as specified in DB links and projects, as well as subsidiaries
 sub _redirect_ln {
     my $dm = shift;
+    my $c = $dm->c;
+    return undef if !$dm->pedantic;
     my ($path, $trailing_slash) = $dm->path;
-    return undef if $trailing_slash || !$dm->pedantic;
 
-    my $c = $dm->c;
-    $c->log->error($c->dumper('DIR::render_ln')) if $MCDEBUG;
-    my ($ln, $etag, $version) = $root->detect_ln_in_the_same_folder($dm->original_path);
+    return undef if $trailing_slash;
+    return undef unless ($dm->extra || $path =~ m/([C|c]urrent)\.iso(\.sha256(\.asc)?)?/);
+    my $original_path = $dm->original_path;
+    my ($ln, $etag, $version) = $root->detect_ln_in_the_same_folder($original_path);
+    return undef if ($ln &&  $dm->extra && !$dm->accept && $original_path ne $path ); # e.g. requested .zsync file and it exists
     my $extra = 1;
-    unless ($ln) {
+    unless ($ln || $original_path eq $path) {
         ($ln, $etag, $version) = $root->detect_ln_in_the_same_folder($path);
         $extra = 0;
     }
@@ -256,7 +259,7 @@ sub _redirect_project_geo {
     my ($path, $trailing_slash) = $dm->path;
 
     my $c = $dm->c;
-    $c->log->error($c->dumper('DIR::render_project_ln_geo')) if $MCDEBUG;
+    $c->log->error($c->dumper('DIR::render_project_geo')) if $MCDEBUG;
     # each project may have a redirect defined in DB, so all requests are redirected for it
     unless ($trailing_slash) {
         my $redirect = $c->mcproject->redirect($path, $dm->region);
@@ -363,7 +366,11 @@ sub _local_render {
     return _render_top_folders($dm) if @top_folders && $path eq '/';
     my $original_path = $dm->original_path;
 
-    return $root->render_file_if_nfs($dm, $path) if $root->is_remote && ($original_path eq $path || (!$dm->extra && !$dm->accept));
+    if ($root->is_remote && ($original_path eq $path || (!$dm->extra && !$dm->accept))) {
+        if ($accept || $dm->must_render_from_root) {
+            return $root->render_file_if_nfs($dm, $path);
+        }
+    }
     return undef if $root->is_remote;
     $c->log->error($c->dumper('local_render3: ')) if $MCDEBUG;
 
@@ -470,7 +477,7 @@ sub _render_from_db {
                     $path = $path . '.zsync';
                 }
 
-                if ($file->{target}) {
+                if ($file->{target} && 1 < index($file->{name}, 'Current')) {
                     # redirect to the symlink
                     my $eq = ($file->{name} eq substr($dm->original_path, -length($file->{name})));
                     $dm->redirect($dm->route . $dirname . '/' . $file->{target}, ($eq && ($dm->accept_all || !$dm->accept)));

lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm

@@ -121,6 +121,23 @@ sub register {
         $fileoriginpath = $realdirname . '/' . $basename if $realdirname ne $dirname;
         return $root->render_file($dm, $fileoriginpath, 1) if $dm->must_render_from_root && !$c->req->headers->if_modified_since; # && $root->is_reachable;
 
+        # check if file is a symlink in the same folder
+        my ($ln, $etag, $version) = ($basename, undef, undef);
+        unless ($root->is_remote) {
+            ($ln, $etag, $version) = $root->detect_ln_in_the_same_folder($dm->original_path);
+            my $extra = 1;
+            unless ($ln) {
+                ($ln, $etag, $version) = $root->detect_ln_in_the_same_folder($filepath);
+                $extra = 0;
+            }
+        }
+        if($ln) {
+            my @arr = split /\//,$ln; # split path
+            $ln = $arr[(scalar(@arr))-1];
+        } else {
+            $ln = $basename;
+        }
+
         if ($folder || $realfolder_id) {
             my $fldid = ($realfolder_id? $realfolder_id : $folder_id);
             $folder_id = $fldid unless $folder_id;
@@ -129,11 +146,11 @@ sub register {
             $x = '.zsync' if  ($dm->zsync && !$dm->accept_zsync);
 
             if (!$dm->zsync) {
-                $file = $schema->resultset('File')->find_with_hash($fldid, $basename, $x) unless $file;
+                $file = $schema->resultset('File')->find_with_hash($fldid, $ln, $x) unless $file;
             } elsif (!$dm->meta4 && !$dm->metalink) {
-                $file = $schema->resultset('File')->find_with_zhash($fldid, $basename, $x);
+                $file = $schema->resultset('File')->find_with_zhash($fldid, $ln, $x);
             } else {
-                $file = $schema->resultset('File')->find_with_hash_and_zhash($fldid, $basename, $x);
+                $file = $schema->resultset('File')->find_with_hash_and_zhash($fldid, $ln, $x);
             }
         }
         my $country = $dm->country;
@@ -150,7 +167,11 @@ sub register {
 
         $c->log->error($c->dumper('RENDER FILE_ID', $file->{id})) if $MCDEBUG;
         $c->res->headers->vary('Accept, COUNTRY, X-COUNTRY, Fastly-SSL');
-        $c->res->headers->etag($dm->etag) if defined $dm->file_size;
+        if ($etag) {
+            $c->res->headers->etag($etag);
+        } elsif (defined $dm->file_size) {
+            $c->res->headers->etag($dm->etag);
+        }
         $c->res->headers->add('X-MEDIA-VERSION' => $dm->media_version) if $dm->media_version;
 
         my $mtime = $file->{mtime};
@@ -203,7 +224,7 @@ sub register {
         my $realproject_id;
         $realproject_id = $c->mcproject->get_id($realdirname) if ($realfolder_id && $realfolder_id != $folder_id);
         my $limit = $dm->mirrorlist ? 300 : (( $dm->metalink || $dm->meta4 || $dm->zsync || $dm->pedantic )? $dm->metalink_limit : 1);
-        my $cnt = _collect_mirrors($dm, \@mirrors_country, \@mirrors_region, \@mirrors_rest, $file->{id}, $file->{name}, $folder_id, $project_id, $realfolder_id, $realproject_id, $limit);
+        my $cnt = _collect_mirrors($dm, \@mirrors_country, \@mirrors_region, \@mirrors_rest, $file->{id}, $basename, $folder_id, $project_id, $realfolder_id, $realproject_id, $limit);
 
         my $mirror;
         $mirror = $mirrors_country[0] if @mirrors_country;

lib/MirrorCache/WebAPI/Plugin/RootLocal.pm

@@ -99,8 +99,8 @@ sub is_dir {
 
 sub render_file {
     my ($self, $dm, $filepath, $not_miss) = @_;
-    my $realpath = $self->realpath($filepath) unless $root_subtree;
-    $filepath = $realpath if $realpath;
+    # my $realpath = $self->realpath($filepath) unless $root_subtree;
+    # $filepath = $realpath if $realpath;
     my $c = $dm->c;
     my $redirect = $self->redirect($dm, $filepath);
     my $res;

t/environ/01-smoke-current.sh

@@ -0,0 +1,46 @@
+#!lib/test-in-container-environ.sh
+set -exo pipefail
+
+mc=$(environ mc $(pwd))
+
+MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=1
+$mc/gen_env MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL \
+            MIRRORCACHE_HASHES_COLLECT=1 \
+            MIRRORCACHE_ZSYNC_COLLECT=dat \
+            MIRRORCACHE_HASHES_PIECES_MIN_SIZE=5
+
+$mc/start
+$mc/status
+
+ap8=$(environ ap8)
+ap7=$(environ ap7)
+
+for x in $mc $ap7 $ap8; do
+    mkdir -p $x/dt/folder1
+    echo 1111111111 > $x/dt/folder1/file1.1.dat
+    echo 1111111111 > $x/dt/folder1/file2.1.dat
+    ( cd $x/dt/folder1/ && ln -s file2.1.dat x-Media.dat )
+done
+
+$ap7/start
+rm $ap7/dt/folder1/file2.1.dat # this mirror is missing the file for whatever reasons
+rm $ap7/dt/folder1/x-Media.dat
+touch $ap7/dt/folder1/x-Media.dat # but still has the symlink
+$ap8/start
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','de','eu'"
+
+# force scan
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/job -e mirror_scan -a '["/folder1"]'
+$mc/backstage/shoot
+
+$mc/curl -IL /download/folder1/x-Media.dat
+rc=0
+$mc/curl -IL /download/folder1/x-Media.dat | grep $($ap7/print_address) || rc=$?
+test $rc -gt 0
+
+$mc/curl -IL /download/folder1/x-Media.dat | grep $($ap8/print_address)
+
+echo success

t/environ/03-headquarter-subsidiaries-hashes.sh

@@ -76,23 +76,24 @@ done
 
 echo Step 3. Add media symlinks and make sure they are imported properly
 for i in 9 6 7 8; do
-    ( cd mc$i/dt/folder1/ && ln -s file4.1.dat file-Media.iso )
+    ( cd mc$i/dt/folder1/ && ln -s file4.1.dat file-Current.iso )
     ( cd mc$i/dt/folder1/ && ln -s file4.1.dat xcurr.dat )
     ( cd mc$i/dt/folder1/ && ln -s file4.1.dat.zsync xcurr.dat.zsync )
     ( cd mc$i/dt/folder1/ && ln -s file4.1.dat xfile-NoCloud.qcow2 )
     mc$i/backstage/job -e folder_sync -a '["/folder1"]'
     mc$i/backstage/shoot
     mc$i/backstage/shoot -q hashes
-    mc$i/sql_test file4.1.dat == "select hash.target from hash join file on id = file_id where name='file-Media.iso'"
-    for x in '' metalink mirrorlist; do
+    mc$i/sql_test file4.1.dat == "select hash.target from hash join file on id = file_id where name='file-Current.iso'"
+    test $i != 9 || mc$i/curl -I /folder1/file-Current.iso | grep -C 10 302 | grep /folder1/file4.1.dat | grep -v /download/folder1/file4.1.dat
+    for x in metalink mirrorlist; do
         ext=""
         [ -z "$x" ] || ext=.$x
-        mc$i/curl -I /folder1/file-Media.iso$ext | grep -C 10 302 | grep /folder1/file4.1.dat$ext | grep -v /download/folder1/file4.1.dat
+        mc$i/curl -I /folder1/file-Current.iso$ext | grep -C 10 302 | grep /folder1/file4.1.dat$ext | grep -v /download/folder1/file4.1.dat
         mc$i/curl -I /folder1/xfile-NoCloud.qcow2$ext | grep -C 10 302 | grep /folder1/file4.1.dat$ext | grep -v /download/folder1/file4.1.dat
     done
 done
 
-mc9/curl -IL /download/folder1/file-Media.iso | grep '200 OK'
+mc9/curl -IL /download/folder1/file-Current.iso | grep '200 OK'
 
 echo Step 4. Add files, but hash calculation on the main server happens later
 DELAY=1;
@@ -114,16 +115,15 @@ done
 
 sleep $DELAY
 
-mc9/curl -I /download/folder1/file-Media.iso            | grep 'Location: /download/folder1/file4.1.dat'
-mc9/curl -I /download/folder1/file-Media.iso.metalink   | grep 'Location: /download/folder1/file4.1.dat\.metalink'
-mc9/curl -I /download/folder1/file-Media.iso?metalink   | grep 'Location: /download/folder1/file4.1.dat?metalink='
-mc9/curl -I /download/folder1/file-Media.iso.mirrorlist | grep 'Location: /download/folder1/file4.1.dat\.mirrorlist'
-mc9/curl -I /download/folder1/file-Media.iso?mirrorlist | grep 'Location: /download/folder1/file4.1.dat?mirrorlist='
+mc9/curl -I /download/folder1/file-Current.iso            | grep 'Location: /download/folder1/file4.1.dat'
+mc9/curl -I /download/folder1/file-Current.iso.metalink   | grep 'Location: /download/folder1/file4.1.dat\.metalink'
+mc9/curl -I /download/folder1/file-Current.iso?metalink   | grep 'Location: /download/folder1/file4.1.dat?metalink='
+mc9/curl -I /download/folder1/file-Current.iso.mirrorlist | grep 'Location: /download/folder1/file4.1.dat\.mirrorlist'
+mc9/curl -I /download/folder1/file-Current.iso?mirrorlist | grep 'Location: /download/folder1/file4.1.dat?mirrorlist='
 
-# mc9/curl -I /download/folder1/xcurr.dat            | grep "Location: http://$na_address/download/folder1/xcurr.dat"
+mc9/curl -I /download/folder1/xcurr.dat            | grep "Location: http://$na_address/download/folder1/xcurr.dat"
 # mc9/curl -I /download/folder1/xcurr.dat.metalink   | grep "Location: http://$na_address/download/folder1/xcurr.dat.metalink"
 # mc9/curl -I /download/folder1/xcurr.dat?meta4      | grep "Location: http://$na_address/download/folder1/xcurr.dat?meta4"
-mc9/curl -I /download/folder1/xcurr.dat            | grep 'Location: /download/folder1/file4.1.dat'
 mc9/curl -I /download/folder1/xcurr.dat.metalink   | grep 'Location: /download/folder1/file4.1.dat.metalink'
 mc9/curl -I /download/folder1/xcurr.dat?meta4      | grep 'Location: /download/folder1/file4.1.dat?meta4'
 mc9/curl -I /download/folder1/xcurr.dat.mirrorlist | grep '/folder1/file4.1.dat.mirrorlist'
@@ -134,7 +134,7 @@ mc9/curl -I /download/folder1/xcurr.dat.zsync.mirrorlist | grep 'Location: /down
 
 mc6/sql_test file4.1.dat == "select target from file where name = 'xcurr.dat'"
 
-mc6/curl -IL /download/folder1/xcurr.dat           | grep '200 OK'
+mc6/curl -I /download/folder1/xcurr.dat            | grep '200 OK'
 mc6/curl -I /download/folder1/xcurr.dat.metalink   | grep '/folder1/file4.1.dat\.metalink'
 mc6/curl -I /download/folder1/xcurr.dat?meta4      | grep '/folder1/file4.1.dat?meta4='
 mc6/curl -I /download/folder1/xcurr.dat.mirrorlist | grep '/folder1/file4.1.dat\.mirrorlist'

t/environ/04-remote-current-no-nfs.sh

@@ -14,16 +14,16 @@ ap7=$(environ ap7)
 
 for x in $ap7 $ap8 $ap9; do
     mkdir -p $x/dt/{folder1,folder2,folder3}
-    echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}-Media.iso | xargs -n 1 touch
-    sha256sum $x/dt/folder1/file1.1-Media.iso > $x/dt/folder1/file1.1-Media.iso.sha256
-    echo 111112 > $x/dt/folder1/file2.1-Media.iso
-    sha256sum $x/dt/folder1/file2.1-Media.iso > $x/dt/folder1/file2.1-Media.iso.sha256
-    ( cd $x/dt/folder1 && ln -s file1.1-Media.iso file-Media.iso && ln -s file1.1-Media.iso.sha256 file-Media.iso.sha256 )
+    echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}-Current.iso | xargs -n 1 touch
+    sha256sum $x/dt/folder1/file1.1-Current.iso > $x/dt/folder1/file1.1-Current.iso.sha256
+    echo 111112 > $x/dt/folder1/file2.1-Current.iso
+    sha256sum $x/dt/folder1/file2.1-Current.iso > $x/dt/folder1/file2.1-Current.iso.sha256
+    ( cd $x/dt/folder1 && ln -s file1.1-Current.iso file-Current.iso && ln -s file1.1-Current.iso.sha256 file-Current.iso.sha256 )
 done
 
 echo '    RewriteEngine On
     RewriteBase "/"
-    RewriteRule ^folder1/file-Media.iso(.*)?$ folder1/file1.1-Media.iso$1 [R]
+    RewriteRule ^folder1/file-Current.iso(.*)?$ folder1/file1.1-Current.iso$1 [R]
 ' > $ap9/directory-rewrite.conf
 
 echo 'LoadModule rewrite_module    /usr/lib64/apache2-prefork/mod_rewrite.so' > $ap9/extra-rewrite.conf
@@ -45,25 +45,25 @@ $mc/backstage/shoot
 $mc/sql "select * from file"
 
 ################################################
-# Test unversioned Media.iso is redirected to file which is metioned inside corresponding Media.iso.sha256
-$mc/curl -I /download/folder1/file-Media.iso        | grep -C 10 302 | grep /download/folder1/file1.1-Media.iso
-$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep /download/folder1/file1.1-Media.iso.sha256
-$mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  "
+# Test unversioned Current.iso is redirected to file which is metioned inside corresponding Current.iso.sha256
+$mc/curl -I /download/folder1/file-Current.iso        | grep -C 10 302 | grep /download/folder1/file1.1-Current.iso
+$mc/curl -I /download/folder1/file-Current.iso.sha256 | grep -C 10 302 | grep /download/folder1/file1.1-Current.iso.sha256
+$mc/curl -L /download/folder1/file-Current.iso.sha256 | grep -q "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  "
 
 echo now change the symlink and make sure redirect changes
 echo '    RewriteEngine On
     RewriteBase "/"
-    RewriteRule ^folder1/file-Media.iso(.*)?$ folder1/file2.1-Media.iso$1 [R]
+    RewriteRule ^folder1/file-Current.iso(.*)?$ folder1/file2.1-Current.iso$1 [R]
 ' > $ap9/directory-rewrite.conf
 
 $ap9/stop
 $ap9/start
 
 $mc/backstage/job -e folder_sync -a '["/folder1"]'
 $mc/backstage/shoot
-$mc/curl -I /download/folder1/file-Media.iso        | grep -C 10 302 | grep /download/folder1/file2.1-Media.iso
-$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep /download/folder1/file2.1-Media.iso.sha256
-$mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q "2019dd7afaf5759c68cec4d0e7553227657f01c69da168489116a1c48e40270e  "
+$mc/curl -I /download/folder1/file-Current.iso        | grep -C 10 302 | grep /download/folder1/file2.1-Current.iso
+$mc/curl -I /download/folder1/file-Current.iso.sha256 | grep -C 10 302 | grep /download/folder1/file2.1-Current.iso.sha256
+$mc/curl -L /download/folder1/file-Current.iso.sha256 | grep -q "2019dd7afaf5759c68cec4d0e7553227657f01c69da168489116a1c48e40270e  "
 
 echo success
 

t/environ/04-remote-current.sh

@@ -24,6 +24,10 @@ for x in $ap7 $ap8 $ap9; do
     ( cd $x/dt/folder1 && ln -s file2.1-Media.iso.zsync file-Media.iso.zsync )
 done
 
+rm $ap7/dt/folder1/file1.1-Media.iso
+rm $ap7/dt/folder1/file2.1-Media.iso.zsync
+rm $ap8/dt/folder1/file1.1-Media.iso.sha256
+
 for x in $ap7 $ap8 $ap9; do
     $x/start
 done
@@ -35,6 +39,7 @@ $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($a
 $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','us','na'"
 
 $mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/job -e mirror_scan -a '["/folder1"]'
 $mc/backstage/shoot
 
 
@@ -43,12 +48,13 @@ $mc/sql_test "select count(*) from file where target is not null"
 
 ################################################
 # Test unversioned Media.iso is redirected to file which is metioned inside corresponding Media.iso.sha256
-$mc/curl -I /download/folder1/file-Media.iso        | grep -C 10 302 | grep /download/folder1/file1.1-Media.iso
-$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep /download/folder1/file1.1-Media.iso.sha256
+$mc/curl -I /download/folder1/file-Media.iso        | grep -C 10 302 # | grep /download/folder1/file1.1-Media.iso
+# $mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep /download/folder1/file1.1-Media.iso.sha256
 $mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  "
 
-$mc/curl -I /download/folder1/file2.1-Media.iso.zsync | grep --color=never -P 'Location: http://127.0.0.1:1324/folder1/file2.1-Media.iso.zsync\r$'
-$mc/curl -I /download/folder1/file-Media.iso.zsync    | grep --color=never -P 'file2.1-Media.iso.zsync\r$'
+$mc/curl -I /download/folder1/file2.1-Media.iso.zsync | grep --color=never -P 'Location: http://127.0.0.1:1314/folder1/file2.1-Media.iso.zsync\r$'
+# $mc/curl -I /download/folder1/file-Media.iso.zsync    | grep --color=never -P 'file2.1-Media.iso.zsync\r$'
+$mc/curl -I /download/folder1/file-Media.iso.zsync  | grep -C10 302 | grep -E "$($ap8/print_address)/folder1/file-Media.iso.zsync|$($ap7/print_address)/folder1/file-Media.iso.zsync"
 
 echo now change the symlink and make sure redirect changes
 (
@@ -57,12 +63,11 @@ echo now change the symlink and make sure redirect changes
     ln -sf file2.1-Media.iso.sha256 file-Media.iso.sha256
 )
 $mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/job -e mirror_scan -a '["/folder1"]'
 $mc/backstage/shoot
-$mc/curl -I /download/folder1/file-Media.iso        | grep -C 10 302 | grep /download/folder1/file2.1-Media.iso
-$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep /download/folder1/file2.1-Media.iso.sha256
+$mc/curl -I /download/folder1/file-Media.iso        | grep -C 10 302 | grep $($ap8/print_address)/folder1/file-Media.iso
+$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep '200 OK'
 $mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q "2019dd7afaf5759c68cec4d0e7553227657f01c69da168489116a1c48e40270e  "
-$mc/curl -I /download/folder1/file-Media.iso        | grep -i Etag
-$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -i Etag
 echo success