Skip to content

Commit

Permalink
Merge pull request #20229 from mimi1vx/pc_slem_6
Browse files Browse the repository at this point in the history 
[PublicCloud] Skip validate repo on SLEM 6+
  • Loading branch information
@asmorodskyi
asmorodskyi authored Oct 16, 2024
2 parents 2f22c89 + 3723bc7 commit 6191d85
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 18 deletions.
4 changes: 4 additions & 0 deletions lib/publiccloud/utils.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,10 @@ sub register_openstack {
# Validation for update repos
sub validate_repo {
my ($maintrepo) = @_;
if (is_sle_micro('>=6.0')) {
record_info("Product Increments", "Can't validate repository");
return 1;
}
if ($maintrepo =~ /\/(PTF|Maintenance):\/(\d+)/g) {
my ($incident, $type) = ($2, $1);
die "We did not detect incident number for URL \"$maintrepo\". We detected \"$incident\"" unless $incident =~ /\d+/;
Expand Down
24 changes: 18 additions & 6 deletions tests/publiccloud/download_repos.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use warnings;
use testapi;
use strict;
use utils;
use version_utils 'is_sle_micro';
use publiccloud::ssh_interactive "select_host_console";
use publiccloud::utils "validate_repo";

Expand Down Expand Up @@ -60,31 +61,42 @@ sub run {
my $count = scalar @repos;
my $check_empty_repos = get_var('PUBLIC_CLOUD_IGNORE_EMPTY_REPO', 0) == 0;
die "No test repositories" if ($check_empty_repos && $count == 0);

my $ret = 0;
my $reject = "'robots.txt,*.ico,*.png,*.gif,*.css,*.js,*.htm*'";
my $reject = "'robots.txt,*.ico,*.png,*.gif,*.css,*.js,*.htm*,*.mirrorlist'";
my $regex = "'s390x\\/|ppc64le\\/|kernel*debuginfo*.rpm|src\\/'";
my ($incident, $type);

set_var("PUBLIC_CLOUD_EMBARGOED_UPDATES_DETECTED", 0);

# workaround for buggy wget
my $cutdirs;
if (is_sle_micro(">=6.0")) {
$cutdirs = '--cut-dirs=5';
}

for my $maintrepo (@repos) {
unless (validate_repo($maintrepo)) {
set_var("PUBLIC_CLOUD_EMBARGOED_UPDATES_DETECTED", 1);
next;
}

script_run("echo 'Downloading $maintrepo ...' >> ~/repos/qem_download_status.txt");
my ($parent) = $maintrepo =~ 'https?://(.*)$';
my ($domain) = $parent =~ '^([a-zA-Z.]*)';
$ret = script_run "wget --no-clobber -r --reject $reject --reject-regex=$regex --domains $domain --no-parent $maintrepo/", timeout => 600;

$ret = script_run "wget $cutdirs --no-clobber -r --reject $reject --reject-regex=$regex --domains $domain --no-parent $maintrepo/", timeout => 600;
if ($ret !~ /0|8/) {
# softfailure, if repo doesn't exist (anymore). This is required for cloning jobs, because the original test repos could be empty already
record_info('Softfail', "Download failed (rc=$ret):\n$maintrepo", result => 'softfail');
record_info('Softfail', "Download /failed (rc=$ret):\n$maintrepo", result => 'softfail');
script_run("echo 'Download failed for $maintrepo ...' >> ~/repos/qem_download_status.txt");
} else {
assert_script_run("echo -en '\\n" . ('#' x 80) . "\\n# $maintrepo:\\n' >> /tmp/repos.list.txt");
assert_script_run("echo 'Downloaded $maintrepo:' \$(du -hs $parent | cut -f1) >> ~/repos/qem_download_status.txt");
if (script_run("ls $parent/*.repo") == 0) {
assert_script_run(sprintf(q(sed -i '1 s/]/_%s]/' %s/*.repo), random_string(4), $parent));
assert_script_run("find $parent >> /tmp/repos.list.txt");
} elsif (is_sle_micro(">=6.0")) {
$parent =~ s/ibs\/SUSE:\/ALP:\/Products:\/Marble:\///;
assert_script_run("find $parent >> /tmp/repos.list.txt");
} else {
record_info('Softfail', "No .repo file found in $parent. This directory will be removed.", result => 'softfail');
assert_script_run("echo 'No .repo found for $maintrepo' >> ~/repos/qem_download_status.txt");
Expand All @@ -97,7 +109,7 @@ sub run {
upload_logs('/tmp/repos.list.txt');
upload_logs('qem_download_status.txt');
# Failsafe 2: Ensure the repos are not empty (i.e. size >= 100 kB)
my $size = script_output('du -s ~/repos | awk \'{print$1}\'');
my $size = script_output(q(du -s ~/repos | awk '{print $1}'));
# we will not die if repos are empty due to embargoed updates filtering
die "Empty test repositories" if (!get_var("PUBLIC_CLOUD_EMBARGOED_UPDATES_DETECTED") && $check_empty_repos && $size < 100);
}
Expand Down
40 changes: 28 additions & 12 deletions tests/publiccloud/transfer_repos.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,14 @@ use strict;
use utils;
use publiccloud::ssh_interactive "select_host_console";
use maintenance_smelt qw(is_embargo_update);
use version_utils qw(is_sle_micro);

sub run {
my ($self, $args) = @_;
select_host_console(); # select console on the host, not the PC instance

my $remote = $args->{my_instance}->username . '@' . $args->{my_instance}->public_ip;
my $instance = $args->{my_instance};
my $remote = $instance->username . '@' . $args->{my_instance}->public_ip;
my @addons = split(/,/, get_var('SCC_ADDONS', ''));
my $skip_mu = get_var('PUBLIC_CLOUD_SKIP_MU', 0);
my $repodir = "/opt/repos/";
Expand All @@ -31,7 +33,7 @@ sub run {
} else {
assert_script_run('du -sh ~/repos');
my $timeout = 2400;
$args->{my_instance}->retry_ssh_command(cmd => "which rsync || sudo zypper -n in rsync", timeout => 420, retry => 6, delay => 60);
$instance->retry_ssh_command(cmd => "which rsync || sudo zypper -n in rsync", timeout => 420, retry => 6, delay => 60);

# In Incidents there is INCIDENT_REPO instead of MAINT_TEST_REPO
# Those two variables contain list of repositories separated by comma
Expand All @@ -48,18 +50,27 @@ sub run {
my @repos;
my ($incident, $type);
for my $maintrepo (@all_repos) {
($incident, $type) = ($2, $1) if ($maintrepo =~ /\/(PTF|Maintenance):\/(\d+)/g);
push(@repos, $maintrepo) unless (is_embargo_update($incident, $type));
if (is_sle_micro(">=6.0")) {
push(@repos, $maintrepo);
} else {
($incident, $type) = ($2, $1) if ($maintrepo =~ /\/(PTF|Maintenance):\/(\d+)/g);
push(@repos, $maintrepo) unless (is_embargo_update($incident, $type)); }
}

s/https?:\/\/// for @repos;

# Create list of directories for rsync
# hold last processed repo in sub scope; for PI its only one repo and we need reuse this var
my $slmrepo;
foreach my $repo (@repos) {
if (is_sle_micro(">=6.0")) {
$repo =~ s/ibs\/SUSE:\/ALP:\/Products:\/Marble:\///;
$slmrepo = $repo;
}
assert_script_run("echo $repo | tee -a /tmp/transfer_repos.txt");
}
# VM repos.dir support preparation
$args->{my_instance}->ssh_assert_script_run("sudo mkdir $repodir;sudo chmod 777 $repodir");
$instance->ssh_assert_script_run("sudo mkdir $repodir;sudo chmod 777 $repodir");
# Mitigate occasional CSP network problems (especially one CSP is prone to those issues!)
# Delay of 2 minutes between the tries to give their network some time to recover after a failure
# For rsync the ~/repos/./ means that the --relative will take efect after.
Expand All @@ -69,16 +80,21 @@ sub run {
# --recursive (-r), --update (-u), --archive (-a), --human-readable (-h), --rsh (-e)
script_retry("rsync --timeout=$timeout -ruahd -e ssh --files-from /tmp/transfer_repos.txt ~/repos/./ '$remote:$repodir'", timeout => $timeout + 10, retry => 3, delay => 120);

my $total_size = $args->{my_instance}->ssh_script_output(cmd => "du -hs $repodir");
my $total_size = $instance->ssh_script_output(cmd => "du -hs $repodir");
record_info("Repo size", "Total repositories size: $total_size");
$args->{my_instance}->ssh_assert_script_run("find $repodir -name '*.rpm' -exec du -h '{}' + | sort -h > /tmp/rpm_list.txt", timeout => 60);
$args->{my_instance}->upload_log('/tmp/rpm_list.txt');
$instance->ssh_assert_script_run("find $repodir -name '*.rpm' -exec du -h '{}' + | sort -h > /tmp/rpm_list.txt", timeout => 60);
$instance->upload_log('/tmp/rpm_list.txt');

$args->{my_instance}->ssh_assert_script_run("sudo find $repodir -name *.repo -exec sed -i 's,http://,$repodir,g' '{}' \\;");
$args->{my_instance}->ssh_assert_script_run("sudo find $repodir -name *.repo -exec zypper ar -p10 '{}' \\;");
$args->{my_instance}->ssh_assert_script_run("sudo find $repodir -name *.repo -exec echo '{}' \\;");
if (is_sle_micro(">=6.0")) {
$instance->ssh_assert_script_run("sudo zypper ar -p10 " . $repodir . $slmrepo . " ToTest");
}
else {
$instance->ssh_assert_script_run("sudo find $repodir -name *.repo -exec sed -i 's,http://,$repodir,g' '{}' \\;");
$instance->ssh_assert_script_run("sudo find $repodir -name *.repo -exec zypper ar -p10 '{}' \\;");
$instance->ssh_assert_script_run("sudo find $repodir -name *.repo -exec echo '{}' \\;");
}

$args->{my_instance}->ssh_assert_script_run("zypper lr -P");
$instance->ssh_assert_script_run("zypper lr -P");
}
}

Expand Down

0 comments on commit 6191d85

Please sign in to comment.