From 5d469009259522e32869600cc0ab4a117194a75c Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 11:16:59 -0400 Subject: [PATCH 01/78] crontab: import from production, except with changes --- cron/CRONTAB.ROOT | 71 ++++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index d141001b2..20cc8753c 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -1,39 +1,42 @@ -MAILTO=andreas.koenig.5c1c1wmb@franz.ak.mind.de -PATH=/opt/perl/current/bin:/usr/bin:/bin:/home/puppet/pause/cron +# MAILTO=andreas.koenig.5c1c1wmb@franz.ak.mind.de -# HEADER: This file was autogenerated at Fri Dec 21 16:00:34 +0000 2012 by puppet. -# HEADER: While it can still be managed manually, it is definitely not recommended. -# HEADER: Note particularly that the comments starting with 'Puppet Name' should -# HEADER: not be deleted, as doing so could cause duplicate cron jobs. -# Puppet Name: puppet-restart -#6 4 * * * sleep $(expr $RANDOM \% 300); /sbin/service puppet restart > /dev/null +PATH=/home/pause/.plenv/shims:/usr/bin:/home/pause/pause/cron +PAUSE_REPO=/home/pause/pause +PAUSE_ROOT=/home/pause/pub/PAUSE -* * * * * recentfile-aggregate.sh -* * * * * date -u +"\%s \%a \%b \%e \%T \%Z \%Y" > /home/ftp/tmp/02STAMP && mv /home/ftp/tmp/02STAMP /home/ftp/pub/PAUSE/authors/02STAMP && /opt/perl/current/bin/perl -I /home/puppet/pause/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' /home/ftp/pub/PAUSE/authors/02STAMP -08 * * * * date -u +"\%s \%FT\%TZ" > /home/ftp/tmp/02STAMPm && mv /home/ftp/tmp/02STAMPm /home/ftp/pub/PAUSE/modules/02STAMP && /opt/perl/current/bin/perl -I /home/puppet/pause/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' /home/ftp/pub/PAUSE/modules/02STAMP -52 * * * * perl /home/puppet/pause/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log -04 7 * * 6 perl /home/puppet/pause/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --symlinkinventory -17,29,41,53 * * * * perl /home/puppet/pause/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite -12 06,14,22 * * * perl /home/puppet/pause/cron/update-checksums.pl -29 * * * * perl /home/puppet/pause/cron/cleanup-incoming.pl -*/3 * * * * perl /home/puppet/pause/cron/cleanup-apachecores.pl -59 * * * * perl /home/puppet/pause/cron/cron-daily.pl -37 05 * * * perl /home/puppet/pause/cron/gmls-lR.pl -47 07,13,19,01 * * * perl /home/puppet/pause/cron/mysql-dump.pl -19 * * * * perl /home/puppet/pause/cron/make-mirror-yaml.pl -26,56 * * * * perl /home/puppet/pause/cron/publish-crontab.pl -21 */6 * * * perl /home/puppet/pause/cron/rm_stale_links -23 07,13,19,01 * * * run_mirrors.sh -22 * * * * perl /home/puppet/pause/cron/sync-04pause.pl -03 07,13,18,01 * * * cd /home/ftp/pub/PAUSE/PAUSE-git && (git gc && git push -u origin master) >> /var/log/git-gc-push.out -4,11,19,26,34,42,49,56 * * * * zsh /home/puppet/pause/cron/assert-paused-running.zsh -18 * * * * perl /home/puppet/pause/cron/cron-p6daily.pl +## STUFF RJBS DID TO PUT THIS INTO UNPAUSE: +## * replace a bunch of paths: +## * /opt/perl/current/bin with /usr/bin/perl +## * put "perl" in front of things to use plenv perl instead of system perl +## * put the pause repo's cron directory in path *and use it* +## +## …and we will write this to /etc/cron.d/SOMETHING -#7 3,9 * * * perl /home/kstar/cron/indexscripts.pl >/dev/null 2>&1 -#7 2 * * 0 perl /home/kstar/cron/indexscripts.pl -f +# ??? +* * * * * root $PAUSE_REPO/cron/recentfile-aggregate.sh -#>19:47:02 root@pause2:/home/puppet/pause# PATH=/opt/perl/current/bin:/usr/bin:/bin:/home/puppet/pause/cron perl -Ilib -c bin/indexscripts.pl -#bin/indexscripts.pl syntax OK +# some kind of PAUSE heartbeat/health check system? +* * * * * root date -u +"\%s \%a \%b \%e \%T \%Z \%Y" > /tmp/02STAMP && mv /tmp/02STAMP $PAUSE_ROOT/authors/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/authors/02STAMP +08 * * * * root date -u +"\%s \%FT\%TZ" > /tmp/02STAMPm && mv /tmp/02STAMPm $PAUSE_ROOT/modules/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/modules/02STAMP + +# THE INDEXER +52 * * * * root perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log +04 7 * * 6 root perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --symlinkinventory +17,29,41,53 * * * * root perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite + +12 06,14,22 * * * root perl $PAUSE_REPO/cron/update-checksums.pl +29 * * * * root perl $PAUSE_REPO/cron/cleanup-incoming.pl +*/3 * * * * root perl $PAUSE_REPO/cron/cleanup-apachecores.pl +59 * * * * root perl $PAUSE_REPO/cron/cron-daily.pl +37 05 * * * root perl $PAUSE_REPO/cron/gmls-lR.pl +47 07,13,19,01 * * * root perl $PAUSE_REPO/cron/mysql-dump.pl +19 * * * * root perl $PAUSE_REPO/cron/make-mirror-yaml.pl +21 */6 * * * root perl $PAUSE_REPO/cron/rm_stale_links +23 07,13,19,01 * * * root perl run_mirrors.sh +22 * * * * root perl $PAUSE_REPO/cron/sync-04pause.pl +10 09,15,21,03 * * * root cd $PAUSE_ROOT/PAUSE-git && (git gc && git push -u origin master) >> /var/log/git-gc-push.out +18 * * * * root perl $PAUSE_REPO/cron/cron-p6daily.pl +46 0,6,12,18 * * * root perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl > $PAUSE_REPO/bin/indexscripts.pl.out 2>&1 +7 2 * * 0 root perl -I $PAUSE_ROOT/lib $PAUSE_ROOT/bin/indexscripts.pl -f +4,11,19,26,34,42,49,56 * * * * root zsh $PAUSE_ROOT/cron/assert-paused-running.zsh -46 0,6,12,18 * * * perl -I/home/puppet/pause/lib /home/puppet/pause/bin/indexscripts.pl >/home/puppet/pause/bin/indexscripts.pl.out 2>&1 -7 2 * * 0 perl -I/home/puppet/pause/lib /home/puppet/pause/bin/indexscripts.pl -f From 3617837ddf8429a2f949d2c5d7a83710642ccf08 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 11:27:33 -0400 Subject: [PATCH 02/78] services: add systemd service definition for paused --- services/paused.service | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 services/paused.service diff --git a/services/paused.service b/services/paused.service new file mode 100644 index 000000000..7ccf17334 --- /dev/null +++ b/services/paused.service @@ -0,0 +1,25 @@ +[Unit] +Description=PAUSE service +Documentation=https://github.com/andk/PAUSE/ + +# Wait for the network to be up +After=network-online.target + +# Require the network service to be present, and at least started at the same time as this service +Wants=network-online.target + +[Service] +Type=simple + +User=pause +Group=pause +WorkingDirectory=/home/pause/pause + +# Should not daemonize +ExecStart=/home/pause/.plenv/shims/perl /home/pause/pause/bin/paused --pidfile /home/pause/pid/paused.pid + +Restart=on-failure +RestartSec=30 + +[Install] +WantedBy=multi-user.target From ca36f23d442fc6c13a8031e5f4fa1d7cb04a40e2 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 12:28:29 -0400 Subject: [PATCH 03/78] services: add pause-web service --- services/pause-web.service | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 services/pause-web.service diff --git a/services/pause-web.service b/services/pause-web.service new file mode 100644 index 000000000..62dd8af89 --- /dev/null +++ b/services/pause-web.service @@ -0,0 +1,25 @@ +[Unit] +Description=PAUSE web service +Documentation=https://github.com/andk/PAUSE/ + +# Wait for the network to be up +After=network-online.target + +# Require the network service to be present, and at least started at the same time as this service +Wants=network-online.target + +[Service] +Type=simple + +User=pause +Group=pause +WorkingDirectory=/home/pause/pause + +# Should not daemonize +ExecStart=/home/pause/.plenv/shims/plackup --port 5000 + +Restart=on-failure +RestartSec=30 + +[Install] +WantedBy=multi-user.target From 121b1f966a2cec292541ebd397c50cdb2c710a06 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 13:18:50 -0400 Subject: [PATCH 04/78] bootstrap/selfconfig: used during "unpause" style bootstrap --- bootstrap/selfconfig | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100755 bootstrap/selfconfig diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig new file mode 100755 index 000000000..ca67f03c1 --- /dev/null +++ b/bootstrap/selfconfig @@ -0,0 +1,70 @@ +#!/usr/bin/bash +set -e + +cd ~pause + +# install plenv so we can manage a local perl version +git clone https://github.com/tokuhirom/plenv.git ~/.plenv + +echo 'export PATH="$HOME/.plenv/bin:$PATH"' >> ~/.bash_profile +echo 'eval "$(plenv init -)"' >> ~/.bash_profile +source ~/.bash_profile + +# install perl-build so we can build a new perl +git clone https://github.com/tokuhirom/Perl-Build.git ~/.plenv/plugins/perl-build/ + +plenv install 5.36.0 -j 8 +plenv global 5.36.0 + +# install cpanm for perl dep management +plenv install-cpanm + +# We need to pin these for now +cpanm -n Mojolicious@8.72 +cpanm -n DBD::mysql@4.052 + +cd ~pause/pause +cpanm -n --installdeps . + +# Set up pause config +mkdir -p ~pause/pause-private/lib + +cat << 'CONF' > ~pause/pause-private/lib/PrivatePAUSE.pm +use strict; +package PAUSE; + +$ENV{EMAIL_SENDER_TRANSPORT} = 'DevNull'; + +our $Config; +$Config->{AUTHEN_DATA_SOURCE_USER} = "pause"; +$Config->{AUTHEN_DATA_SOURCE_PW} = "pausepassword"; +$Config->{MOD_DATA_SOURCE_USER} = "pause"; +$Config->{MOD_DATA_SOURCE_PW} = "pausepassword"; +$Config->{MAIL_MAILER} = ["testfile"]; +$Config->{RUNDATA} = "/tmp/pause_1999"; + +$Config->{CHECKSUMS_SIGNING_PROGRAM} = "does-not-exist"; +$Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; +$Config->{MLROOT} = '/home/pause/pub/PAUSE/authors/id/'; +$Config->{ML_CHOWN_USER} = 'unsafe'; +$Config->{ML_CHOWN_GROUP} = 'unsafe'; +$Config->{ML_MIN_FILES} = 1; +$Config->{ML_MIN_INDEX_LINES} = 0; +$Config->{PAUSE_LOG} = "/home/pause/log/paused.log"; +$Config->{PAUSE_LOG_DIR} = "/home/pause/log/"; +$Config->{PID_DIR} = "/home/pause/pid/"; +$Config->{TMP} = "/tmp/"; +CONF + +mkdir ~pause/log +mkdir ~pause/pid +mkdir -p ~pause/pub/PAUSE/authors/id +mkdir -p ~pause/pub/PAUSE/modules +mkdir -p ~pause/pub/PAUSE/PAUSE-git + +cd ~pause/pub/PAUSE/PAUSE-git +git init +git config --global user.email "pause@pause.perl.org" +git config --global user.name "PAUSE Daemon" + +mkdir -p /tmp/pause_1999 From 97dc23d765afba05a9aacfc1b8089635ebdf870f Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 22:06:32 -0400 Subject: [PATCH 05/78] cron/recentfile-aggregate.sh: update PATH --- cron/recentfile-aggregate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron/recentfile-aggregate.sh b/cron/recentfile-aggregate.sh index 18d4eaffb..c34b22dcd 100755 --- a/cron/recentfile-aggregate.sh +++ b/cron/recentfile-aggregate.sh @@ -1,5 +1,5 @@ -#!/bin/sh +#!/bin/bash -export PATH=/opt/perl/current/bin:/usr/local/perl/bin:$PATH +export PATH=/home/pause/.plenv/shims:$PATH rrr-aggregate /home/ftp/pub/PAUSE/authors/RECENT-1h.yaml rrr-aggregate /home/ftp/pub/PAUSE/modules/RECENT-1h.yaml From 969919abd728ad40a484019cbff8eb10afd85fd8 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 22:12:59 -0400 Subject: [PATCH 06/78] crontab: run pause jobs as pause user --- cron/CRONTAB.ROOT | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index 20cc8753c..8857e4c22 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -13,30 +13,30 @@ PAUSE_ROOT=/home/pause/pub/PAUSE ## …and we will write this to /etc/cron.d/SOMETHING # ??? -* * * * * root $PAUSE_REPO/cron/recentfile-aggregate.sh +* * * * * pause $PAUSE_REPO/cron/recentfile-aggregate.sh # some kind of PAUSE heartbeat/health check system? -* * * * * root date -u +"\%s \%a \%b \%e \%T \%Z \%Y" > /tmp/02STAMP && mv /tmp/02STAMP $PAUSE_ROOT/authors/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/authors/02STAMP -08 * * * * root date -u +"\%s \%FT\%TZ" > /tmp/02STAMPm && mv /tmp/02STAMPm $PAUSE_ROOT/modules/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/modules/02STAMP +* * * * * pause date -u +"\%s \%a \%b \%e \%T \%Z \%Y" > /tmp/02STAMP && mv /tmp/02STAMP $PAUSE_ROOT/authors/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/authors/02STAMP +08 * * * * pause date -u +"\%s \%FT\%TZ" > /tmp/02STAMPm && mv /tmp/02STAMPm $PAUSE_ROOT/modules/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/modules/02STAMP # THE INDEXER -52 * * * * root perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log -04 7 * * 6 root perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --symlinkinventory -17,29,41,53 * * * * root perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite +52 * * * * pause perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log +04 7 * * 6 pause perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --symlinkinventory +17,29,41,53 * * * * pause perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite -12 06,14,22 * * * root perl $PAUSE_REPO/cron/update-checksums.pl -29 * * * * root perl $PAUSE_REPO/cron/cleanup-incoming.pl -*/3 * * * * root perl $PAUSE_REPO/cron/cleanup-apachecores.pl -59 * * * * root perl $PAUSE_REPO/cron/cron-daily.pl -37 05 * * * root perl $PAUSE_REPO/cron/gmls-lR.pl -47 07,13,19,01 * * * root perl $PAUSE_REPO/cron/mysql-dump.pl -19 * * * * root perl $PAUSE_REPO/cron/make-mirror-yaml.pl -21 */6 * * * root perl $PAUSE_REPO/cron/rm_stale_links -23 07,13,19,01 * * * root perl run_mirrors.sh -22 * * * * root perl $PAUSE_REPO/cron/sync-04pause.pl -10 09,15,21,03 * * * root cd $PAUSE_ROOT/PAUSE-git && (git gc && git push -u origin master) >> /var/log/git-gc-push.out -18 * * * * root perl $PAUSE_REPO/cron/cron-p6daily.pl -46 0,6,12,18 * * * root perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl > $PAUSE_REPO/bin/indexscripts.pl.out 2>&1 -7 2 * * 0 root perl -I $PAUSE_ROOT/lib $PAUSE_ROOT/bin/indexscripts.pl -f -4,11,19,26,34,42,49,56 * * * * root zsh $PAUSE_ROOT/cron/assert-paused-running.zsh +12 06,14,22 * * * pause perl $PAUSE_REPO/cron/update-checksums.pl +29 * * * * pause perl $PAUSE_REPO/cron/cleanup-incoming.pl +*/3 * * * * pause perl $PAUSE_REPO/cron/cleanup-apachecores.pl +59 * * * * pause perl $PAUSE_REPO/cron/cron-daily.pl +37 05 * * * pause perl $PAUSE_REPO/cron/gmls-lR.pl +47 07,13,19,01 * * * pause perl $PAUSE_REPO/cron/mysql-dump.pl +19 * * * * pause perl $PAUSE_REPO/cron/make-mirror-yaml.pl +21 */6 * * * pause perl $PAUSE_REPO/cron/rm_stale_links +23 07,13,19,01 * * * pause perl run_mirrors.sh +22 * * * * pause perl $PAUSE_REPO/cron/sync-04pause.pl +10 09,15,21,03 * * * pause cd $PAUSE_ROOT/PAUSE-git && (git gc && git push -u origin master) >> /var/log/git-gc-push.out +18 * * * * pause perl $PAUSE_REPO/cron/cron-p6daily.pl +46 0,6,12,18 * * * pause perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl > $PAUSE_REPO/bin/indexscripts.pl.out 2>&1 +7 2 * * 0 pause perl -I $PAUSE_ROOT/lib $PAUSE_ROOT/bin/indexscripts.pl -f +4,11,19,26,34,42,49,56 * * * * pause zsh $PAUSE_ROOT/cron/assert-paused-running.zsh From 869e91dd422ec27dc2acc0de76d653731a73acec Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 22:16:06 -0400 Subject: [PATCH 07/78] cron: remove cleanup-apachecores.pl We no longer use Apache! --- cron/CRONTAB.ROOT | 1 - cron/cleanup-apachecores.pl | 97 ------------------------------------- 2 files changed, 98 deletions(-) delete mode 100755 cron/cleanup-apachecores.pl diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index 8857e4c22..989d3ee1f 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -26,7 +26,6 @@ PAUSE_ROOT=/home/pause/pub/PAUSE 12 06,14,22 * * * pause perl $PAUSE_REPO/cron/update-checksums.pl 29 * * * * pause perl $PAUSE_REPO/cron/cleanup-incoming.pl -*/3 * * * * pause perl $PAUSE_REPO/cron/cleanup-apachecores.pl 59 * * * * pause perl $PAUSE_REPO/cron/cron-daily.pl 37 05 * * * pause perl $PAUSE_REPO/cron/gmls-lR.pl 47 07,13,19,01 * * * pause perl $PAUSE_REPO/cron/mysql-dump.pl diff --git a/cron/cleanup-apachecores.pl b/cron/cleanup-apachecores.pl deleted file mode 100755 index ed18c90ea..000000000 --- a/cron/cleanup-apachecores.pl +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/perl - -# use 5.010; -use strict; -use warnings; - -=head1 NAME - - - -=head1 SYNOPSIS - - - -=head1 OPTIONS - -=over 8 - -=cut - -my @opt = <<'=back' =~ /B<--(\S+)>/g; - -=item B<--coredir=s> - -Directory to cleanup. Defaults to /opt/apache/cores. - -=item B<--filerx=s> - -Regular expression that filters files subject to the cleanup. Defaults to C - -=item B<--help|h!> - -This help - -=item B<--keep=i> - -Number of files to keep. Defaults to 10. - -=item B<--verbose!> - -Report about deletions. - -=back - -=head1 DESCRIPTION - - - -=cut - - -use FindBin; -use lib "$FindBin::Bin/../lib"; -BEGIN { - push @INC, qw( ); -} - -use Dumpvalue; -use File::Basename qw(dirname); -use File::Path qw(mkpath); -use File::Spec; -use File::Temp; -use Getopt::Long; -use Pod::Usage; -use Hash::Util qw(lock_keys); - -our %Opt; -lock_keys %Opt, map { /([^=|!]+)/ } @opt; -GetOptions(\%Opt, - @opt, - ) or pod2usage(1); - -$Opt{coredir} ||= "/opt/apache/cores"; -$Opt{keep} ||= 10; -$Opt{filerx} ||= q{^core\.\d+\z}; -my $filerxqr = qr{$Opt{filerx}}; -opendir my $dh, $Opt{coredir} or die "Could not open '$Opt{coredir}': $!"; -my @ls; -for my $dirent (readdir $dh) { - next unless $dirent =~ $filerxqr; - push @ls, "$Opt{coredir}/$dirent"; -} -my @ls_sorted_newest_first = map { $_->[0] } - sort { $a->[1] <=> $b->[1] } - map { [$_, -M $_] } - @ls; -while (@ls_sorted_newest_first > $Opt{keep}) { - my $unlink = pop @ls_sorted_newest_first; - print STDERR "About to delete '$unlink'..." if $Opt{verbose}; - unlink $unlink or die "Could not unlink '$unlink': $!"; - print STDERR "Done\n" if $Opt{verbose}; -} - -# Local Variables: -# mode: cperl -# cperl-indent-level: 4 -# End: From 128a1bce4a447843fc902529662fca9d18d38321 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 22:21:17 -0400 Subject: [PATCH 08/78] cron programs: use plenv-based perl shebang --- cron/cron-daily.pl | 3 ++- cron/cron-p6daily.pl | 2 +- cron/gmls-lR.pl | 3 ++- cron/make-mirror-yaml.pl | 2 +- cron/mldistwatch | 2 +- cron/mysql-dump.pl | 3 ++- cron/publish-crontab.pl | 2 +- cron/restart-httpd | 2 +- cron/rm_stale_links | 3 ++- cron/run_mirrors.sh | 2 +- cron/sync-04pause.pl | 2 +- cron/update-checksums.pl | 2 +- 12 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cron/cron-daily.pl b/cron/cron-daily.pl index afba5c4bc..e0792954a 100755 --- a/cron/cron-daily.pl +++ b/cron/cron-daily.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl use FindBin; use lib "$FindBin::Bin/../lib"; @@ -16,6 +16,7 @@ use Email::Sender::Simple (); use strict; +use warnings; use vars qw( $last_str $last_time $SUBJECT @listing $Dbh); # diff --git a/cron/cron-p6daily.pl b/cron/cron-p6daily.pl index 48e1c3689..18966881c 100755 --- a/cron/cron-p6daily.pl +++ b/cron/cron-p6daily.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl use FindBin; use lib "$FindBin::Bin/../lib"; diff --git a/cron/gmls-lR.pl b/cron/gmls-lR.pl index 1af734d82..d107f13a9 100755 --- a/cron/gmls-lR.pl +++ b/cron/gmls-lR.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl =comment @@ -18,6 +18,7 @@ use PAUSE (); use File::Compare qw(compare); use strict; +use warnings; chdir $PAUSE::Config->{FTPPUB} or die "Could not chdir to $PAUSE::Config->{FTPPUB}: $!"; mkdir "indexes", 0755 unless -d "indexes"; diff --git a/cron/make-mirror-yaml.pl b/cron/make-mirror-yaml.pl index b75e38441..d5fca21c6 100755 --- a/cron/make-mirror-yaml.pl +++ b/cron/make-mirror-yaml.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/home/pause/.plenv/shims/perl use strict; use warnings; diff --git a/cron/mldistwatch b/cron/mldistwatch index b187f38ea..99e447b8a 100755 --- a/cron/mldistwatch +++ b/cron/mldistwatch @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/home/pause/.plenv/shims/perl =head1 NAME diff --git a/cron/mysql-dump.pl b/cron/mysql-dump.pl index 08fc1c42e..d494cf002 100755 --- a/cron/mysql-dump.pl +++ b/cron/mysql-dump.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl =pod @@ -15,6 +15,7 @@ =cut use strict; +use warnings; use FindBin; use lib "$FindBin::Bin/../lib"; diff --git a/cron/publish-crontab.pl b/cron/publish-crontab.pl index 96671171e..431966181 100755 --- a/cron/publish-crontab.pl +++ b/cron/publish-crontab.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/home/pause/.plenv/shims/perl die " # since we are switching to use /etc/cron.d/pause2016 for the core diff --git a/cron/restart-httpd b/cron/restart-httpd index ceeb716cf..e4ca87fd5 100755 --- a/cron/restart-httpd +++ b/cron/restart-httpd @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/home/pause/.plenv/shims/perl # scheduled for demise as soon as the pause2.develooper.com has taken over diff --git a/cron/rm_stale_links b/cron/rm_stale_links index dab51d0f2..3dcceb0ec 100755 --- a/cron/rm_stale_links +++ b/cron/rm_stale_links @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w -- -*- mode: cperl -*- +#!/home/pause/.plenv/shims/perl =pod @@ -18,6 +18,7 @@ use lib "$FindBin::Bin/../lib"; use PAUSE (); use strict; +use warnings; use File::Find; use File::Spec; diff --git a/cron/run_mirrors.sh b/cron/run_mirrors.sh index a6c73f190..b498f6ded 100755 --- a/cron/run_mirrors.sh +++ b/cron/run_mirrors.sh @@ -11,4 +11,4 @@ else exit 1 fi -perl /usr/bin/mirror -C$MIRRDIR/$MIRRCNF $MIRRDIR/mymirror.config +/home/pause/.plenv/shims/perl /usr/bin/mirror -C$MIRRDIR/$MIRRCNF $MIRRDIR/mymirror.config diff --git a/cron/sync-04pause.pl b/cron/sync-04pause.pl index f9e1197b5..200a194c4 100755 --- a/cron/sync-04pause.pl +++ b/cron/sync-04pause.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/home/pause/.plenv/shims/perl use FindBin; use lib "$FindBin::Bin/../lib"; diff --git a/cron/update-checksums.pl b/cron/update-checksums.pl index 7606a1a0b..ffe731b2c 100755 --- a/cron/update-checksums.pl +++ b/cron/update-checksums.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl # use 5.010; use strict; From 788b67ce1d8d4afec402731a938431e8c5e11464 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 22:22:56 -0400 Subject: [PATCH 09/78] cron: remove assert-paused-running.zsh systemd will handle this. --- cron/CRONTAB.ROOT | 2 -- cron/assert-paused-running.zsh | 5 ----- 2 files changed, 7 deletions(-) delete mode 100755 cron/assert-paused-running.zsh diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index 989d3ee1f..97b8f1f51 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -37,5 +37,3 @@ PAUSE_ROOT=/home/pause/pub/PAUSE 18 * * * * pause perl $PAUSE_REPO/cron/cron-p6daily.pl 46 0,6,12,18 * * * pause perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl > $PAUSE_REPO/bin/indexscripts.pl.out 2>&1 7 2 * * 0 pause perl -I $PAUSE_ROOT/lib $PAUSE_ROOT/bin/indexscripts.pl -f -4,11,19,26,34,42,49,56 * * * * pause zsh $PAUSE_ROOT/cron/assert-paused-running.zsh - diff --git a/cron/assert-paused-running.zsh b/cron/assert-paused-running.zsh deleted file mode 100755 index 362944134..000000000 --- a/cron/assert-paused-running.zsh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/zsh - -if ! /etc/init.d/PAUSE-paused status > /dev/null ; then - /etc/init.d/PAUSE-paused start -fi From 9efec956ae0a8d7b4182331fbbe7462a2017f009 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 5 Apr 2024 22:39:17 -0400 Subject: [PATCH 10/78] cron: remove "perl" from programs with shebang and +x --- cron/CRONTAB.ROOT | 26 +++++++++++++------------- cron/cleanup-incoming.pl | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index 97b8f1f51..a65ccd16e 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -20,20 +20,20 @@ PAUSE_ROOT=/home/pause/pub/PAUSE 08 * * * * pause date -u +"\%s \%FT\%TZ" > /tmp/02STAMPm && mv /tmp/02STAMPm $PAUSE_ROOT/modules/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/modules/02STAMP # THE INDEXER -52 * * * * pause perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log -04 7 * * 6 pause perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --symlinkinventory -17,29,41,53 * * * * pause perl $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite +52 * * * * pause $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log +04 7 * * 6 pause $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --symlinkinventory +17,29,41,53 * * * * pause $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite -12 06,14,22 * * * pause perl $PAUSE_REPO/cron/update-checksums.pl -29 * * * * pause perl $PAUSE_REPO/cron/cleanup-incoming.pl -59 * * * * pause perl $PAUSE_REPO/cron/cron-daily.pl -37 05 * * * pause perl $PAUSE_REPO/cron/gmls-lR.pl -47 07,13,19,01 * * * pause perl $PAUSE_REPO/cron/mysql-dump.pl -19 * * * * pause perl $PAUSE_REPO/cron/make-mirror-yaml.pl -21 */6 * * * pause perl $PAUSE_REPO/cron/rm_stale_links -23 07,13,19,01 * * * pause perl run_mirrors.sh -22 * * * * pause perl $PAUSE_REPO/cron/sync-04pause.pl +12 06,14,22 * * * pause $PAUSE_REPO/cron/update-checksums.pl +29 * * * * pause $PAUSE_REPO/cron/cleanup-incoming.pl +59 * * * * pause $PAUSE_REPO/cron/cron-daily.pl +37 05 * * * pause $PAUSE_REPO/cron/gmls-lR.pl +47 07,13,19,01 * * * pause $PAUSE_REPO/cron/mysql-dump.pl +19 * * * * pause $PAUSE_REPO/cron/make-mirror-yaml.pl +21 */6 * * * pause $PAUSE_REPO/cron/rm_stale_links +23 07,13,19,01 * * * pause $PAUSE_REPO/cron/run_mirrors.sh +22 * * * * pause $PAUSE_REPO/cron/sync-04pause.pl 10 09,15,21,03 * * * pause cd $PAUSE_ROOT/PAUSE-git && (git gc && git push -u origin master) >> /var/log/git-gc-push.out -18 * * * * pause perl $PAUSE_REPO/cron/cron-p6daily.pl +18 * * * * pause $PAUSE_REPO/cron/cron-p6daily.pl 46 0,6,12,18 * * * pause perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl > $PAUSE_REPO/bin/indexscripts.pl.out 2>&1 7 2 * * 0 pause perl -I $PAUSE_ROOT/lib $PAUSE_ROOT/bin/indexscripts.pl -f diff --git a/cron/cleanup-incoming.pl b/cron/cleanup-incoming.pl index 3cde9fa37..d2233493f 100755 --- a/cron/cleanup-incoming.pl +++ b/cron/cleanup-incoming.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/home/pause/.plenv/shims/perl use FindBin; use lib "$FindBin::Bin/../lib"; From 0e26e0d8950dfd6909d7574ab8af7813e7880d81 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 10:17:15 -0400 Subject: [PATCH 11/78] bootstrap: switch to perl because it makes rjbs more comfortable and because he wants a real getopt --- bootstrap/selfconfig | 128 ++++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 56 deletions(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index ca67f03c1..711f5ae6b 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -1,70 +1,86 @@ -#!/usr/bin/bash -set -e +#!/usr/bin/perl +use v5.36.0; -cd ~pause +use Carp qw(croak); +use Getopt::Long::Descriptive; +use Path::Tiny; + +chdir("/home/pause/pause") or die "can't chdir to ~pause/pause: $!"; # install plenv so we can manage a local perl version -git clone https://github.com/tokuhirom/plenv.git ~/.plenv +run_cmd(qw( + git clone https://github.com/tokuhirom/plenv.git /home/pause/.plenv +)); -echo 'export PATH="$HOME/.plenv/bin:$PATH"' >> ~/.bash_profile -echo 'eval "$(plenv init -)"' >> ~/.bash_profile -source ~/.bash_profile +path("/home/pause/.bash_profile")->append(<<~'END'); + export PATH="$HOME/.plenv/bin:$PATH" + eval "$(plenv init -)" + END # install perl-build so we can build a new perl -git clone https://github.com/tokuhirom/Perl-Build.git ~/.plenv/plugins/perl-build/ +run_cmd(qw( + git clone https://github.com/tokuhirom/Perl-Build.git + /home/pause/.plenv/plugins/perl-build/ +)); -plenv install 5.36.0 -j 8 -plenv global 5.36.0 +run_cmd(qw( /home/pause/.plenv/bin/plenv install 5.36.0 -j 8 )); +run_cmd(qw( /home/pause/.plenv/bin/plenv global 5.36.0 )); # install cpanm for perl dep management -plenv install-cpanm +run_cmd(qw( /home/pause/.plenv/bin/plenv install-cpanm )); # We need to pin these for now -cpanm -n Mojolicious@8.72 -cpanm -n DBD::mysql@4.052 +run_cmd(qw( /home/pause/.plenv/shims/cpanm -n Mojolicious@8.72 )); +run_cmd(qw( /home/pause/.plenv/shims/cpanm -n DBD::mysql@4.052 )); -cd ~pause/pause -cpanm -n --installdeps . +run_cmd(qw( /home/pause/.plenv/shims/cpanm -n --installdeps . )); # Set up pause config -mkdir -p ~pause/pause-private/lib - -cat << 'CONF' > ~pause/pause-private/lib/PrivatePAUSE.pm -use strict; -package PAUSE; - -$ENV{EMAIL_SENDER_TRANSPORT} = 'DevNull'; - -our $Config; -$Config->{AUTHEN_DATA_SOURCE_USER} = "pause"; -$Config->{AUTHEN_DATA_SOURCE_PW} = "pausepassword"; -$Config->{MOD_DATA_SOURCE_USER} = "pause"; -$Config->{MOD_DATA_SOURCE_PW} = "pausepassword"; -$Config->{MAIL_MAILER} = ["testfile"]; -$Config->{RUNDATA} = "/tmp/pause_1999"; - -$Config->{CHECKSUMS_SIGNING_PROGRAM} = "does-not-exist"; -$Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; -$Config->{MLROOT} = '/home/pause/pub/PAUSE/authors/id/'; -$Config->{ML_CHOWN_USER} = 'unsafe'; -$Config->{ML_CHOWN_GROUP} = 'unsafe'; -$Config->{ML_MIN_FILES} = 1; -$Config->{ML_MIN_INDEX_LINES} = 0; -$Config->{PAUSE_LOG} = "/home/pause/log/paused.log"; -$Config->{PAUSE_LOG_DIR} = "/home/pause/log/"; -$Config->{PID_DIR} = "/home/pause/pid/"; -$Config->{TMP} = "/tmp/"; -CONF - -mkdir ~pause/log -mkdir ~pause/pid -mkdir -p ~pause/pub/PAUSE/authors/id -mkdir -p ~pause/pub/PAUSE/modules -mkdir -p ~pause/pub/PAUSE/PAUSE-git - -cd ~pause/pub/PAUSE/PAUSE-git -git init -git config --global user.email "pause@pause.perl.org" -git config --global user.name "PAUSE Daemon" - -mkdir -p /tmp/pause_1999 +for my $path (qw( + /home/pause/log + /home/pause/pause-private/lib + /home/pause/pid + /home/pause/pub/PAUSE/authors/id + /home/pause/pub/PAUSE/modules + /home/pause/pub/PAUSE/PAUSE-git + /tmp/pause_1999 +)) { + path($path)->mkdir; +} + +run_cmd(qw(git init)); +run_cmd(qw(git config --global user.email pause@pause.perl.org)); +run_cmd(qw(git config --global user.name), 'PAUSE Daemon'); + +path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); + use strict; + package PAUSE; + + $ENV{EMAIL_SENDER_TRANSPORT} = 'DevNull'; + + our $Config; + $Config->{AUTHEN_DATA_SOURCE_USER} = "pause"; + $Config->{AUTHEN_DATA_SOURCE_PW} = "pausepassword"; + $Config->{MOD_DATA_SOURCE_USER} = "pause"; + $Config->{MOD_DATA_SOURCE_PW} = "pausepassword"; + $Config->{MAIL_MAILER} = ["testfile"]; + $Config->{RUNDATA} = "/tmp/pause_1999"; + + $Config->{CHECKSUMS_SIGNING_PROGRAM} = "does-not-exist"; + $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; + $Config->{MLROOT} = '/home/pause/pub/PAUSE/authors/id/'; + $Config->{ML_CHOWN_USER} = 'unsafe'; + $Config->{ML_CHOWN_GROUP} = 'unsafe'; + $Config->{ML_MIN_FILES} = 1; + $Config->{ML_MIN_INDEX_LINES} = 0; + $Config->{PAUSE_LOG} = "/home/pause/log/paused.log"; + $Config->{PAUSE_LOG_DIR} = "/home/pause/log/"; + $Config->{PID_DIR} = "/home/pause/pid/"; + $Config->{TMP} = "/tmp/"; + END + +sub run_cmd (@args) { + system {$args[0]} @args; + + croak "failed to run $args[0]" if $?; +} From 983fd6b2646b6ad0a8116be279438717e878b3a9 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 11:25:50 -0400 Subject: [PATCH 12/78] crontab: fix env var typo: ROOT used instead of REPO --- cron/CRONTAB.ROOT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index a65ccd16e..358df8223 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -36,4 +36,4 @@ PAUSE_ROOT=/home/pause/pub/PAUSE 10 09,15,21,03 * * * pause cd $PAUSE_ROOT/PAUSE-git && (git gc && git push -u origin master) >> /var/log/git-gc-push.out 18 * * * * pause $PAUSE_REPO/cron/cron-p6daily.pl 46 0,6,12,18 * * * pause perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl > $PAUSE_REPO/bin/indexscripts.pl.out 2>&1 -7 2 * * 0 pause perl -I $PAUSE_ROOT/lib $PAUSE_ROOT/bin/indexscripts.pl -f +7 2 * * 0 pause perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl -f From feac2be5abc7f4ea149977dab4bd047a6a06cab0 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 11:31:26 -0400 Subject: [PATCH 13/78] bootstrap config: set FTPPUB config option --- bootstrap/selfconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 711f5ae6b..ce1cf5c51 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -67,6 +67,7 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $Config->{RUNDATA} = "/tmp/pause_1999"; $Config->{CHECKSUMS_SIGNING_PROGRAM} = "does-not-exist"; + $Config->{FTPPUB} = '/home/pause/pub/PAUSE/'; $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; $Config->{MLROOT} = '/home/pause/pub/PAUSE/authors/id/'; $Config->{ML_CHOWN_USER} = 'unsafe'; From 26b2738dca42fc3e7b7b79b9cc1a3782a511868b Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 11:31:40 -0400 Subject: [PATCH 14/78] cron/update-checksums.pl: use ~pause/run for run files --- bootstrap/selfconfig | 1 + cron/update-checksums.pl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index ce1cf5c51..433a85b02 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -43,6 +43,7 @@ for my $path (qw( /home/pause/pub/PAUSE/authors/id /home/pause/pub/PAUSE/modules /home/pause/pub/PAUSE/PAUSE-git + /home/pause/run /tmp/pause_1999 )) { path($path)->mkdir; diff --git a/cron/update-checksums.pl b/cron/update-checksums.pl index ffe731b2c..f0b03cd38 100755 --- a/cron/update-checksums.pl +++ b/cron/update-checksums.pl @@ -60,7 +60,7 @@ =head1 DESCRIPTION use Hash::Util qw(lock_keys); use Pod::Usage; -my $lockfile = "/var/run/PAUSE-update-checksums.LCK"; +my $lockfile = "/home/pause/run/PAUSE-update-checksums.LCK"; use Fcntl qw( :flock :seek O_RDONLY O_RDWR O_CREAT ); my $lfh; unless (open $lfh, "+<", $lockfile) { From e5b8bebc786e7db06756e826b6ddeb9bdea8ea0d Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 11:37:09 -0400 Subject: [PATCH 15/78] indexscripts.pl: adjust shebang ...but probably this should be removed --- bin/indexscripts.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/indexscripts.pl b/bin/indexscripts.pl index ef70cd2f2..3cc7a48cf 100755 --- a/bin/indexscripts.pl +++ b/bin/indexscripts.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/home/pause/.plenv/shims/perl # Build the scripts index for PAUSE # Original author: KSTAR # Last modified: $Date: 2003/12/13 05:52:51 $ From 118784a6cf46734ac44ffc788bc7e8d2d5afb971 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 13:33:47 -0400 Subject: [PATCH 16/78] bootstrap/import-mod-db: import production data --- bootstrap/import-mod-db | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 bootstrap/import-mod-db diff --git a/bootstrap/import-mod-db b/bootstrap/import-mod-db new file mode 100755 index 000000000..22802f767 --- /dev/null +++ b/bootstrap/import-mod-db @@ -0,0 +1,4 @@ +#!/bin/bash +rsync -vaP pause.perl.org::pausedata/moddump.current.bz2 . +bzcat moddump.current.bz2 | perl -ple 's/^CHANGE MASTER.*//' | \ + sudo mysql -p mod From 5be572876aa619957878f7457c1bb1894c069393 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 14:28:44 -0400 Subject: [PATCH 17/78] cron/cron-daily.pl: put tmpfile in /tmp Otherwise, it goes into the cwd of the cron process, which is ~pause, and this is weird. --- cron/cron-daily.pl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cron/cron-daily.pl b/cron/cron-daily.pl index e0792954a..65c8f24bf 100755 --- a/cron/cron-daily.pl +++ b/cron/cron-daily.pl @@ -29,6 +29,8 @@ $TIME[5] += 1900; my $TIME = sprintf "%02d" x 5, @TIME[ 5, 4, 3, 2, 1 ]; +my $TMPFILE = "/tmp/00whois.new"; + my $zcat = $PAUSE::Config->{ZCAT_PATH}; die "no executable zcat" unless -x $zcat; my $gzip = $PAUSE::Config->{GZIP_PATH}; @@ -395,7 +397,7 @@ sub whois { scalar(gmtime), $0, ); - open FH, ">00whois.new" or return "Error: Can't open 00whois.new: $!"; + open FH, ">", $TMPFILE or return "Error: Can't open $TMPFILE: $!"; if ($] > 5.007) { require Encode; binmode FH, ":utf8"; @@ -575,10 +577,10 @@ sub whois { ]; close FH; - if (compare '00whois.new', "$PAUSE::Config->{MLROOT}/../00whois.html") { - report qq[copy 00whois.new $PAUSE::Config->{MLROOT}/../00whois.html\n\n]; + if (compare $TMPFILE, "$PAUSE::Config->{MLROOT}/../00whois.html") { + report qq[copy $TMPFILE $PAUSE::Config->{MLROOT}/../00whois.html\n\n]; my $whois_file = "$PAUSE::Config->{MLROOT}/../00whois.html"; - xcopy '00whois.new', $whois_file; + xcopy $TMPFILE, $whois_file; PAUSE::newfile_hook($whois_file); my $xml_file = "$PAUSE::Config->{MLROOT}/../00whois.xml"; From 6894118c0ccdaa5e9b3bd16e677a601089a36258 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 14:34:45 -0400 Subject: [PATCH 18/78] cron/recentfile-aggregate: convert to perl, fix paths (by "fix" I mean "do not hardcode") --- cron/CRONTAB.ROOT | 2 +- cron/recentfile-aggregate | 22 ++++++++++++++++++++++ cron/recentfile-aggregate.sh | 5 ----- 3 files changed, 23 insertions(+), 6 deletions(-) create mode 100755 cron/recentfile-aggregate delete mode 100755 cron/recentfile-aggregate.sh diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index 358df8223..081118ea1 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -13,7 +13,7 @@ PAUSE_ROOT=/home/pause/pub/PAUSE ## …and we will write this to /etc/cron.d/SOMETHING # ??? -* * * * * pause $PAUSE_REPO/cron/recentfile-aggregate.sh +* * * * * pause $PAUSE_REPO/cron/recentfile-aggregate # some kind of PAUSE heartbeat/health check system? * * * * * pause date -u +"\%s \%a \%b \%e \%T \%Z \%Y" > /tmp/02STAMP && mv /tmp/02STAMP $PAUSE_ROOT/authors/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/authors/02STAMP diff --git a/cron/recentfile-aggregate b/cron/recentfile-aggregate new file mode 100755 index 000000000..6ec7fc198 --- /dev/null +++ b/cron/recentfile-aggregate @@ -0,0 +1,22 @@ +#!/home/pause/.plenv/shims/perl +use v5.36.0; + +use PAUSE; +use Path::Tiny; + +my $PAUSE_ROOT = $PAUSE::Config->{FTPPUB}; + +die "FTPPUB directory ($PAUSE_ROOT) isn't there!\n" + unless -d $PAUSE_ROOT; + +my $root = path($PAUSE_ROOT); + +system( + '/home/pause/.plenv/shims/rrr-aggregate', + $root->child('authors/RECENT-1h.yaml'), +); + +system( + '/home/pause/.plenv/shims/rrr-aggregate', + $root->child('modules/RECENT-1h.yaml'), +); diff --git a/cron/recentfile-aggregate.sh b/cron/recentfile-aggregate.sh deleted file mode 100755 index c34b22dcd..000000000 --- a/cron/recentfile-aggregate.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -export PATH=/home/pause/.plenv/shims:$PATH -rrr-aggregate /home/ftp/pub/PAUSE/authors/RECENT-1h.yaml -rrr-aggregate /home/ftp/pub/PAUSE/modules/RECENT-1h.yaml From 773716904d1ca28c37b640ecaaed23d6cc2753cd Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 15:02:55 -0400 Subject: [PATCH 19/78] bootstrap: set up INCOMING_LOC (I am not entirely sure what this is for, but we need it.) --- bootstrap/selfconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 433a85b02..c1b682290 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -37,6 +37,7 @@ run_cmd(qw( /home/pause/.plenv/shims/cpanm -n --installdeps . )); # Set up pause config for my $path (qw( + /home/pause/incoming /home/pause/log /home/pause/pause-private/lib /home/pause/pid @@ -70,6 +71,7 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $Config->{CHECKSUMS_SIGNING_PROGRAM} = "does-not-exist"; $Config->{FTPPUB} = '/home/pause/pub/PAUSE/'; $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; + $Config->{INCOMING_LOC} = '/home/pause/incoming' $Config->{MLROOT} = '/home/pause/pub/PAUSE/authors/id/'; $Config->{ML_CHOWN_USER} = 'unsafe'; $Config->{ML_CHOWN_GROUP} = 'unsafe'; From 6f08cdbac8cc38e7bdf4a5744a3137961ee28df6 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 15:03:14 -0400 Subject: [PATCH 20/78] bootstrap: set CHECKSUMS_SIGNING_ARGS to placeholder --- bootstrap/selfconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index c1b682290..26677abd9 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -69,6 +69,8 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $Config->{RUNDATA} = "/tmp/pause_1999"; $Config->{CHECKSUMS_SIGNING_PROGRAM} = "does-not-exist"; + $Config->{CHECKSUMS_SIGNING_ARGS} = "..."; + $Config->{FTPPUB} = '/home/pause/pub/PAUSE/'; $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; $Config->{INCOMING_LOC} = '/home/pause/incoming' From bfbf499ad4bbc4a6691300b3d28504bb61c28159 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 6 Apr 2024 15:04:20 -0400 Subject: [PATCH 21/78] crontab: log mldistwatch to ~pause/log for now (this should move to journald probably) --- cron/CRONTAB.ROOT | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index 081118ea1..248649c52 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -20,9 +20,9 @@ PAUSE_ROOT=/home/pause/pub/PAUSE 08 * * * * pause date -u +"\%s \%FT\%TZ" > /tmp/02STAMPm && mv /tmp/02STAMPm $PAUSE_ROOT/modules/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/modules/02STAMP # THE INDEXER -52 * * * * pause $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log -04 7 * * 6 pause $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --symlinkinventory -17,29,41,53 * * * * pause $PAUSE_REPO/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite +52 * * * * pause $PAUSE_REPO/cron/mldistwatch --logfile /home/pause/log/mldistwatch.cron.log +04 7 * * 6 pause $PAUSE_REPO/cron/mldistwatch --logfile /home/pause/log/mldistwatch.cron.log --symlinkinventory +17,29,41,53 * * * * pause $PAUSE_REPO/cron/mldistwatch --logfile /home/pause/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite 12 06,14,22 * * * pause $PAUSE_REPO/cron/update-checksums.pl 29 * * * * pause $PAUSE_REPO/cron/cleanup-incoming.pl From de250acdab2bcfd9237b75aa51389706ee946d58 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 19 Apr 2024 13:38:40 -0400 Subject: [PATCH 22/78] bootstrap: fix a missing ; in PAUSE config --- bootstrap/selfconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 26677abd9..fc2ab5197 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -73,7 +73,7 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $Config->{FTPPUB} = '/home/pause/pub/PAUSE/'; $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; - $Config->{INCOMING_LOC} = '/home/pause/incoming' + $Config->{INCOMING_LOC} = '/home/pause/incoming'; $Config->{MLROOT} = '/home/pause/pub/PAUSE/authors/id/'; $Config->{ML_CHOWN_USER} = 'unsafe'; $Config->{ML_CHOWN_GROUP} = 'unsafe'; From deab82ba5d23145f09af64cb14111888208e8ad7 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 19 Apr 2024 13:57:45 -0400 Subject: [PATCH 23/78] bootstrap: chdir to PAUSE-git before git init --- bootstrap/selfconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index fc2ab5197..8c3e2d09a 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -50,6 +50,9 @@ for my $path (qw( path($path)->mkdir; } +chdir("/home/pause/pub/PAUSE/PAUSE-git") + || die "couldn't chdir to PAUSE-git: $!"; + run_cmd(qw(git init)); run_cmd(qw(git config --global user.email pause@pause.perl.org)); run_cmd(qw(git config --global user.name), 'PAUSE Daemon'); From 034993272ad4d534890a1caa5c634466381440b3 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 19 Apr 2024 14:15:02 -0400 Subject: [PATCH 24/78] bootstrap: import database faster --- bootstrap/import-mod-db | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bootstrap/import-mod-db b/bootstrap/import-mod-db index 22802f767..7dc5fd3fc 100755 --- a/bootstrap/import-mod-db +++ b/bootstrap/import-mod-db @@ -1,4 +1,11 @@ #!/bin/bash rsync -vaP pause.perl.org::pausedata/moddump.current.bz2 . -bzcat moddump.current.bz2 | perl -ple 's/^CHANGE MASTER.*//' | \ - sudo mysql -p mod +bunzip2 moddump.current.bz2 + +echo 'SET GLOBAL innodb_flush_log_at_trx_commit=2' | mysql + +pv moddump.current | \ + perl -ple 's/^CHANGE MASTER.*//' | \ + sudo mysql mod + +echo 'SET GLOBAL innodb_flush_log_at_trx_commit=1' | mysql From f009df1e08e36ed5a61f1648115f1104c6f9fb3d Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 19 Apr 2024 14:32:42 -0400 Subject: [PATCH 25/78] bootstrap: compile with more processes --- bootstrap/selfconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 8c3e2d09a..7c80108c2 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -23,7 +23,7 @@ run_cmd(qw( /home/pause/.plenv/plugins/perl-build/ )); -run_cmd(qw( /home/pause/.plenv/bin/plenv install 5.36.0 -j 8 )); +run_cmd(qw( /home/pause/.plenv/bin/plenv install 5.36.0 -j 16 )); run_cmd(qw( /home/pause/.plenv/bin/plenv global 5.36.0 )); # install cpanm for perl dep management From f7dc46e07fd2a248514c158bea80925ecacf8c7c Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 19 Apr 2024 14:33:19 -0400 Subject: [PATCH 26/78] bootstrap: do not install man pages for plenv perl --- bootstrap/selfconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 7c80108c2..88779c42c 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -23,7 +23,7 @@ run_cmd(qw( /home/pause/.plenv/plugins/perl-build/ )); -run_cmd(qw( /home/pause/.plenv/bin/plenv install 5.36.0 -j 16 )); +run_cmd(qw( /home/pause/.plenv/bin/plenv install 5.36.0 -j 16 --noman )); run_cmd(qw( /home/pause/.plenv/bin/plenv global 5.36.0 )); # install cpanm for perl dep management From 6974754e0476e8347b53b78baeeb99780c99ee55 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 19 Apr 2024 14:43:25 -0400 Subject: [PATCH 27/78] bootstrap: provide initial branch name for PAUSE-git --- bootstrap/selfconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 88779c42c..3c0405aec 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -53,9 +53,9 @@ for my $path (qw( chdir("/home/pause/pub/PAUSE/PAUSE-git") || die "couldn't chdir to PAUSE-git: $!"; -run_cmd(qw(git init)); run_cmd(qw(git config --global user.email pause@pause.perl.org)); run_cmd(qw(git config --global user.name), 'PAUSE Daemon'); +run_cmd(qw(git init --initial-branch master )); path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); use strict; From 7b1483afd1398d70dbc64b40616b8d4499397e45 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 19 Apr 2024 15:15:38 -0400 Subject: [PATCH 28/78] bootstrap: learn to use prepackaged plenv --- bootstrap/selfconfig | 70 ++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 3c0405aec..d724c3b48 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -5,35 +5,49 @@ use Carp qw(croak); use Getopt::Long::Descriptive; use Path::Tiny; -chdir("/home/pause/pause") or die "can't chdir to ~pause/pause: $!"; - -# install plenv so we can manage a local perl version -run_cmd(qw( - git clone https://github.com/tokuhirom/plenv.git /home/pause/.plenv -)); - -path("/home/pause/.bash_profile")->append(<<~'END'); - export PATH="$HOME/.plenv/bin:$PATH" - eval "$(plenv init -)" - END - -# install perl-build so we can build a new perl -run_cmd(qw( - git clone https://github.com/tokuhirom/Perl-Build.git - /home/pause/.plenv/plugins/perl-build/ -)); - -run_cmd(qw( /home/pause/.plenv/bin/plenv install 5.36.0 -j 16 --noman )); -run_cmd(qw( /home/pause/.plenv/bin/plenv global 5.36.0 )); - -# install cpanm for perl dep management -run_cmd(qw( /home/pause/.plenv/bin/plenv install-cpanm )); - -# We need to pin these for now -run_cmd(qw( /home/pause/.plenv/shims/cpanm -n Mojolicious@8.72 )); -run_cmd(qw( /home/pause/.plenv/shims/cpanm -n DBD::mysql@4.052 )); +if (-e '/tmp/plenv-tarball.tar.bz2') { + chdir("/home/pause") or die "can't chdir to ~pause: $!"; + + # Somebody helpfully gave us a plenv directory to start from! Let's use it. + run_cmd(qw(tar jxvf /tmp/plenv-tarball.tar.bz2)); + + path("/home/pause/.bash_profile")->append(<<~'END'); + export PATH="$HOME/.plenv/bin:$PATH" + eval "$(plenv init -)" + END +} else { + chdir("/home/pause/pause") or die "can't chdir to ~pause/pause: $!"; + + # install plenv so we can manage a local perl version + run_cmd(qw( + git clone https://github.com/tokuhirom/plenv.git /home/pause/.plenv + )); + + path("/home/pause/.bash_profile")->append(<<~'END'); + export PATH="$HOME/.plenv/bin:$PATH" + eval "$(plenv init -)" + END + + # install perl-build so we can build a new perl + run_cmd(qw( + git clone https://github.com/tokuhirom/Perl-Build.git + /home/pause/.plenv/plugins/perl-build/ + )); + + run_cmd(qw( /home/pause/.plenv/bin/plenv install 5.36.0 -j 16 --noman )); + run_cmd(qw( /home/pause/.plenv/bin/plenv global 5.36.0 )); + + # install cpanm for perl dep management + run_cmd(qw( /home/pause/.plenv/bin/plenv install-cpanm )); + + # We need to pin these for now + run_cmd(qw( /home/pause/.plenv/shims/cpanm -n Mojolicious@8.72 )); + run_cmd(qw( /home/pause/.plenv/shims/cpanm -n DBD::mysql@4.052 )); + + run_cmd(qw( /home/pause/.plenv/shims/cpanm -n --installdeps . )); +} -run_cmd(qw( /home/pause/.plenv/shims/cpanm -n --installdeps . )); +chdir("/home/pause/pause") or die "can't chdir to ~pause/pause: $!"; # Set up pause config for my $path (qw( From d040f8acbf9ac651deb01d6c265f3c51c977e5ae Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 11:10:22 +0100 Subject: [PATCH 29/78] bootstrap: Config: set a AUTHEN_BACKUP_DIR at least for now --- bootstrap/selfconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index d724c3b48..face15af0 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -78,6 +78,7 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $ENV{EMAIL_SENDER_TRANSPORT} = 'DevNull'; our $Config; + $Config->{AUTHEN_BACKUP_DIR} = "/home/pause/db-backup"; $Config->{AUTHEN_DATA_SOURCE_USER} = "pause"; $Config->{AUTHEN_DATA_SOURCE_PW} = "pausepassword"; $Config->{MOD_DATA_SOURCE_USER} = "pause"; From 0a1a511433734d7302bae4b1b9a2c8d3ae4c705e Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 11:11:21 +0100 Subject: [PATCH 30/78] mysql-dump.pl: db is not replicated, do not act as if it is With master=>1 the program died because it was not running binlogging. --- cron/mysql-dump.pl | 1 - 1 file changed, 1 deletion(-) diff --git a/cron/mysql-dump.pl b/cron/mysql-dump.pl index d494cf002..a5dcac459 100755 --- a/cron/mysql-dump.pl +++ b/cron/mysql-dump.pl @@ -42,7 +42,6 @@ cfg_dsn => "MOD_DATA_SOURCE_NAME", cfg_user => "MOD_DATA_SOURCE_USER", cfg_pw => "MOD_DATA_SOURCE_PW", - master => 1, }, {backupdir => $PAUSE::Config->{AUTHEN_BACKUP_DIR}, cfg_dsn => "AUTHEN_DATA_SOURCE_NAME", From 5921644ea32462c68152a995152de91184cf7b33 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 11:16:51 +0100 Subject: [PATCH 31/78] cron/rm_stale_links: never remove critical authors/id dir --- cron/rm_stale_links | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cron/rm_stale_links b/cron/rm_stale_links index 3dcceb0ec..e04f4f595 100755 --- a/cron/rm_stale_links +++ b/cron/rm_stale_links @@ -24,11 +24,23 @@ use File::Find; use File::Spec; chdir "$PAUSE::Config->{MLROOT}../.."; +my %KEEP_FOREVER = ( + # No matter what, do not delete these paths! For an example of why this is + # interesting, imagine a freshly installed PAUSE that has no files uploaded + # yet. Without protecting authors/id, it could be deleted. Later, something + # will try to upload authors/id/X/XY/XYZZY/Foo-1.23.tar.gz, but it can't, + # because authors/id has been removed. Possibly it should be willing to + # create the full tree, but right now, it can't. So, we will avoid deleting + # it instead. + 'authors/id' => 1, +); + find( { bydepth => 1, wanted => sub { return if /^\.\.?$/; + return if $KEEP_FOREVER{ $File::Find::name }; my($dev,$ino,$mode,$nlink,$uid,$gid) = lstat($_); if (-l $_ && ! -e $_){ warn "unlinking stale $File::Find::name\n"; From 51135a0dddb61653c7bd2de0677fa2bb11ad3deb Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 11:49:07 +0100 Subject: [PATCH 32/78] cron-p6daily.pl: put a domain on From address --- cron/cron-p6daily.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/cron-p6daily.pl b/cron/cron-p6daily.pl index 18966881c..86d4cad15 100755 --- a/cron/cron-p6daily.pl +++ b/cron/cron-p6daily.pl @@ -67,7 +67,7 @@ sub send_the_mail { header_str => [ Subject => $SUBJECT, To => $PAUSE::Config->{ADMIN}, - From => "cron daemon cron-p6daily.pl ", + From => "cron daemon cron-p6daily.pl ", ], body_str => join(q{}, @blurb), ); From 2b088f4b5f3f0be12166e91d28265ef505a40f57 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 11:57:49 +0100 Subject: [PATCH 33/78] bootstrap: also set PAUSE_PUBLIC_DATA --- bootstrap/selfconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index face15af0..38e6b7a35 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -99,6 +99,7 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $Config->{ML_MIN_INDEX_LINES} = 0; $Config->{PAUSE_LOG} = "/home/pause/log/paused.log"; $Config->{PAUSE_LOG_DIR} = "/home/pause/log/"; + $Config->{PAUSE_PUBLIC_DATA} = '/home/pause/pub/PAUSE/PAUSE-data'; $Config->{PID_DIR} = "/home/pause/pid/"; $Config->{TMP} = "/tmp/"; END From b3b0d5e28b8ea8e1daa8b3437144c35dfefc3923 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 11:57:59 +0100 Subject: [PATCH 34/78] cron/cron-daily.pl: put a domain on From address --- cron/cron-daily.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/cron-daily.pl b/cron/cron-daily.pl index 65c8f24bf..2d2f70419 100755 --- a/cron/cron-daily.pl +++ b/cron/cron-daily.pl @@ -340,7 +340,7 @@ sub send_the_mail { header_str => [ Subject => $SUBJECT, To => $PAUSE::Config->{ADMIN}, - From => "cron daemon cron-daily.pl ", + From => "cron daemon cron-daily.pl ", ], body_str => join(q{}, @blurb), ); From f61260f1baf05efb49238c7b91d1de13799d3dd0 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 12:43:33 +0100 Subject: [PATCH 35/78] bootstrap/import-mod-db: stop services while mirroring --- bootstrap/import-mod-db | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bootstrap/import-mod-db b/bootstrap/import-mod-db index 7dc5fd3fc..bfee9d8ef 100755 --- a/bootstrap/import-mod-db +++ b/bootstrap/import-mod-db @@ -2,6 +2,10 @@ rsync -vaP pause.perl.org::pausedata/moddump.current.bz2 . bunzip2 moddump.current.bz2 +systemctl stop paused + +sudo -u pause rsync --progress -av pause.perl.org::PAUSE/ /home/pause/pub/PAUSE/ + echo 'SET GLOBAL innodb_flush_log_at_trx_commit=2' | mysql pv moddump.current | \ @@ -9,3 +13,5 @@ pv moddump.current | \ sudo mysql mod echo 'SET GLOBAL innodb_flush_log_at_trx_commit=1' | mysql + +systemctl start paused From 18bb4f54a5078b665aca2112adb6f252d59ba1db Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 13:33:01 +0100 Subject: [PATCH 36/78] recentfile-aggregate: use FindBin to find library dir --- cron/recentfile-aggregate | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cron/recentfile-aggregate b/cron/recentfile-aggregate index 6ec7fc198..48dbe830f 100755 --- a/cron/recentfile-aggregate +++ b/cron/recentfile-aggregate @@ -1,6 +1,9 @@ #!/home/pause/.plenv/shims/perl use v5.36.0; +use FindBin; +use lib "$FindBin::Bin/../lib"; + use PAUSE; use Path::Tiny; From 74640d9a0fc1c1f98f39e00ea05592938af398e6 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 13:35:59 +0100 Subject: [PATCH 37/78] cron/make-mirror-yaml: get rid of it This built the mirror list, which is now a fixed list. --- cron/CRONTAB.ROOT | 1 - cron/make-mirror-yaml.pl | 22 ---------------------- 2 files changed, 23 deletions(-) delete mode 100755 cron/make-mirror-yaml.pl diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index 248649c52..48c107ba9 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -29,7 +29,6 @@ PAUSE_ROOT=/home/pause/pub/PAUSE 59 * * * * pause $PAUSE_REPO/cron/cron-daily.pl 37 05 * * * pause $PAUSE_REPO/cron/gmls-lR.pl 47 07,13,19,01 * * * pause $PAUSE_REPO/cron/mysql-dump.pl -19 * * * * pause $PAUSE_REPO/cron/make-mirror-yaml.pl 21 */6 * * * pause $PAUSE_REPO/cron/rm_stale_links 23 07,13,19,01 * * * pause $PAUSE_REPO/cron/run_mirrors.sh 22 * * * * pause $PAUSE_REPO/cron/sync-04pause.pl diff --git a/cron/make-mirror-yaml.pl b/cron/make-mirror-yaml.pl deleted file mode 100755 index d5fca21c6..000000000 --- a/cron/make-mirror-yaml.pl +++ /dev/null @@ -1,22 +0,0 @@ -#!/home/pause/.plenv/shims/perl - -use strict; -use warnings; -use CPAN::Indexer::Mirror 0.05; # atomic writes -use File::Path qw(mkpath); -use LWP::UserAgent; - -use FindBin; -use lib "$FindBin::Bin/../lib"; -use PAUSE; - -die "FTP_RUN not defined" unless defined $PAUSE::Config->{FTP_RUN}; -my $rundir = "$PAUSE::Config->{FTP_RUN}/mirroryaml"; -mkpath $rundir; -my $ua = LWP::UserAgent->new(agent => "PAUSE/20080922"); -my $resp = $ua->mirror($PAUSE::Config->{MIRRORED_BY_URL},"$rundir/MIRRORED.BY"); -die "Could not mirror: ".$resp->status_line unless $resp->is_success || 304 eq $resp->code; - -CPAN::Indexer::Mirror->new( - root => $rundir, - )->run; From af17ec810d2863441bcb9298c245b9fecddec821 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 13:37:59 +0100 Subject: [PATCH 38/78] bootstrap: put TMP back in ~pause/tmp like production --- bootstrap/selfconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 38e6b7a35..ba0693e17 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -59,6 +59,7 @@ for my $path (qw( /home/pause/pub/PAUSE/modules /home/pause/pub/PAUSE/PAUSE-git /home/pause/run + /home/pause/tmp /tmp/pause_1999 )) { path($path)->mkdir; @@ -101,7 +102,7 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $Config->{PAUSE_LOG_DIR} = "/home/pause/log/"; $Config->{PAUSE_PUBLIC_DATA} = '/home/pause/pub/PAUSE/PAUSE-data'; $Config->{PID_DIR} = "/home/pause/pid/"; - $Config->{TMP} = "/tmp/"; + $Config->{TMP} = "/home/pause/tmp/"; END sub run_cmd (@args) { From 673ec3294ede2679c75618830f12156bed830d58 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 14:11:32 +0100 Subject: [PATCH 39/78] bootstrap/import-mod-db: stop indexing during run --- bootstrap/import-mod-db | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bootstrap/import-mod-db b/bootstrap/import-mod-db index bfee9d8ef..c736be831 100755 --- a/bootstrap/import-mod-db +++ b/bootstrap/import-mod-db @@ -1,7 +1,14 @@ #!/bin/bash +if [ $UID != "0" ]; then + echo "import-mod-db is meant to be run as root" >&2 + exit 1; +fi + rsync -vaP pause.perl.org::pausedata/moddump.current.bz2 . bunzip2 moddump.current.bz2 +touch /etc/PAUSE.CLOSED + systemctl stop paused sudo -u pause rsync --progress -av pause.perl.org::PAUSE/ /home/pause/pub/PAUSE/ @@ -15,3 +22,5 @@ pv moddump.current | \ echo 'SET GLOBAL innodb_flush_log_at_trx_commit=1' | mysql systemctl start paused + +rm /etc/PAUSE.CLOSED From 5987e0afec5f8f130b7a9dbff9090de43efc3913 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 15:35:06 +0100 Subject: [PATCH 40/78] bootstrap: set up a GPG key to sign with --- bootstrap/selfconfig | 9 +++-- bootstrap/test-key.txt | 81 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 bootstrap/test-key.txt diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index ba0693e17..8e6c71428 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -72,6 +72,10 @@ run_cmd(qw(git config --global user.email pause@pause.perl.org)); run_cmd(qw(git config --global user.name), 'PAUSE Daemon'); run_cmd(qw(git init --initial-branch master )); +# This imports a test key, which nobody should trust, and which has key id +# 6BA1716EFB099DB2. -- rjbs, 2024-04-25 +run_cmd(qw( gpg --import --armor /home/pause/pause/bootstrap/test-key.txt )); + path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); use strict; package PAUSE; @@ -87,8 +91,9 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $Config->{MAIL_MAILER} = ["testfile"]; $Config->{RUNDATA} = "/tmp/pause_1999"; - $Config->{CHECKSUMS_SIGNING_PROGRAM} = "does-not-exist"; - $Config->{CHECKSUMS_SIGNING_ARGS} = "..."; + $Config->{CHECKSUMS_SIGNING_PROGRAM} = "gpg"; + $Config->{CHECKSUMS_SIGNING_ARGS} = '--homedir /home/pause/pause-private/gnupg-pause-batch-signing-home --clearsign --default-key '; + $Config->{CHECKSUMS_SIGNING_KEY} = '6BA1716EFB099DB2'; $Config->{FTPPUB} = '/home/pause/pub/PAUSE/'; $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; diff --git a/bootstrap/test-key.txt b/bootstrap/test-key.txt new file mode 100644 index 000000000..d91256660 --- /dev/null +++ b/bootstrap/test-key.txt @@ -0,0 +1,81 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQVYBGYqZ6kBDADhY91ivv6YZ4pNswsL9iJDuwX/8Gge/G7nMNNRkxVlyRPpfUlt +23EDHW/FWj8fJqN/sHWoc8MavXE8A4mTe93x/5kmayjywbJJeDTJujcxCthdpzrL +8tFtD6GwmLnqdjxrJgYeTDfu5cy5upeocQFtd02Gy6J+z21x2EdxVHwjoO4cMvfQ +TvlQ2w6vkxK5cxKk3fXNoCd/+efaBVaPp2yjCXkMEGKIoyoJNXa7q5TL7tRotFUE +vSl00dIvvpxG/s7XRJbjuEfi4Hso5CI3y9gg3yFDC/aS5NsBjIRIEqMk5/5lYPw+ +bwP4fBzLbah3i/q10D+vpTaiU+ELcP04Nhe4pKhfSQEg2/+tX5OIXj0ZHN5hI04w +J09qbto3WolPnhW/Tmm0a9UQljNRcHs0nUTY+MFkwTN1XCLKVyYKjRWKrEtz6buO +Hj6bKDSIgPaZJQXBR0DktMdRbmDXepYtFu30DClj0NMGAwObcvzutZu09IgYbmLP +JNSp6USxU9d4vkcAEQEAAQAL+gIdDHLlR6FnXrQraK92Ao8FZrtHdJ87Ph0h3iFp +AcMByBOEoIPiCMOub3ty1/Niz0Zg7tR45bcRpFLd1Y0R/TihuRuHupAbJVm+55Kh +MxKWC1GI6CWNLf1xnHgkXUwnxvh7WrQJZdq5PS2QTOUhXKl9sGZS1GXR+k1SfIxr +7hwSAr+By/RimOXuZwjoLcIopFGArEiXDNj0tzqvhsIjFpE8Skz5DFwT+flIwxN2 +85a9cNthYOusnzr86+FJb6tOPTRDLaAt6vdi/L1YJAJMkOix7kIXfIn6UqJpc6ld +ugz9+C6CLrZUMhQeZa1prm+lIQ7/o/Pzg5t9MW7Nq/+0HSQMDgQ86UZDG2k5Unvi +cFX4B53qKVBHqVTMR01IhH9SFxj1kwOht6pn7xShy45liC3rZJCE/nXL8wyqNmmU +FWCtFTOWgJlHLpCLKRb4XlSfSpSUPxoOQMxYyUr3p0mUeZYwE7XHAFrNK98IkiVf +5eNqU0v05feegNL/vVxIzvqqaQYA5l2t5+a2XnF5JvGOkVUVwLcrW8+XQ2xbk2XA +Uc465+rMJEa/9dq7Mjd0hXCOcT9HMGqAFsIYHtbYuzQBzLfLdL0kRo3M6o6F6CAA +LCrB+P0Kd1MhxCgNHEGNA9lIK6ySUSjdeStesN5E54RjkKJxEFKAZci45dIwTgC/ +OE2gHSYab25IJkfRENMThlonf0ubUj1rXemYqG1QGgrH4PGybzNlHVOQj/VgEJQb +ozNjkjlLeaGoATUoHLV9CqtcKZRJBgD6eHD0O5RqG8eCtgTIaBO85SacIp0b/x67 +0mrQW/Z5+FlIVVnbAy94v9EPoXDz2cH+c+DM0X+W5VrjmZkNaKOh5i2U+EH4wMlo +D/gB+7ztuI0ceRV9DGd5eGe+XGLxSxCHIZn0TJi6d4mlFsJPQ0t95/StEvZ6cm24 +hA2/ApKGsHp4vqxAXLkGEbRES9X+Za+Txtg7WKWS0DrBKXfsUxA2Ws2rnPrcmrs9 +iw0C+cRGWCKTMXrMkVINyfeMBQNkng8F/1ZtF/afuQ5d2Uc0Q0QWaJJx3LvBLmBU +ycZdihxEwhtDhu4rDyEKVJt+Whu3NsjeLZQ8MNXh3sdTyWNsRX6N+ZCD2kRB4OuJ +d1aOgX9eVQKIABrvXE3IrOLArsHMUkjefa/qmIGRz5nXzQp+/6VtBwI1Gtqe6ahV +iUU4XPRuzbg7UJYwO+PsMVb0TQTUhne5HdTSyJDtFcIbAVGfWwP45/dOuPtKVBkX +EtUhW507zIkh6qIdauxo7z3KBqEg4p0+/dgRtBt0ZXN0aW5nLW9ubHlAcGF1c2Uu +cGVybC5vcmeJAdQEEwEKAD4WIQTpXI0CYrueC+0CKA5roXFu+wmdsgUCZipnqQIb +AwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBroXFu+wmdsr1PDACO +pEYS3uC3HYX88nurkEsQejOjGb3cNGPcMDwJ4ISUQp4upVR3o4QL9eBUwQEp4ERi +9zaLSFFoqhjZLtohrfZ9RFjgmKnYExnxbgNss2JuYnrB97cBt9CSVyemKBdQfoKu +/nwr8FejoZAdsOwth+FzvFX7/pIgylye8P4GQ/818NBIi/uTa3mz3t5mrAQdcPUY +z37rb5CiuOFZHYVPVTK/JqImcCXtuwk7aM3RU/L88WmWZf3Bl9G2k3YUY2cxAcVQ +eScs3an8E6XKCIe6OfEd4jXD3V9us0+77e7DpgHO4Hdtz7kbsgrN+odgij3/cjuX +3lXgjgeEdt6X15ilb4/vzs7c99XXXxqlPOFL3ArCxbQWexLggrS+muNvm4QxEE5+ +2BShEK5d4pBPc8hBUg40iQKyo5BJDCPqSHeVM9EZzcC6twa+ynhn5W9wu4iytLad +FpaUhXLbe4EuDsPKHUsY9gpkcsMzxfiL1iF1rzqiWJAcf2NRuE4ybwvMSCLzyGad +BVgEZipnqQEMANupLB1isvRXitZbZO+8fD01jAFkUXts+pfjmR/lmETk0DqXwThO +5irR+o0lymGxPIwMa5w2wcJC73r+L4l6VM296wkz7SArfUvcRGmPp2F4iWeca6i3 +IFDpgH6FsN+lLUWCugrvE78pPx2mWOyk3n4beizOe8IuH8iSnsqvm5OVEaH/66HE +9NWwVqC9HBSFBzyDqrb+suT0sitHPUPjJQh9nYi3JspblwcCrFLsykPGzSi7x/l2 +JMJbfQy2s/1w+PmT3t0NAB0LyABmsI+qZHjYiRqCIfgeeQPLmK2LcWyZ4y+sLcC/ +vwIPx5NPXZGSf6tjqkAE2ik7XXBP2294ozaWfiVYxgBwUdUBaX/deGz5a3s3WVPy +ARxexsiCgHtJm2/jgYdkBFusfT845xYj8ExwiunEeAQ3+ddbKlEg/7XK969bWdLP +xSjelyehsHuyg7YroT57ePX6RAV3LC+S/k0mRRMmdMetMcf1bbU4yZiJAg1zNbba +HKa/8XNkHvhpTwARAQABAAv9EKPeKFO5GXLQR7DLaAXapjW6PMNsEMeOrEGc8KJP +SK4fasY4PZsXK0NA5xRnzRgaGBnmGlgOPDh0ioKwTd/V23LPAqF2Yyt9kGjyncnj +dNTRbpHrmJpWzl+spj5bycq/hTaEAB61sOjfqSVYMy/CKGzNx+XUe5ShlRYrr/sM +SWp6LXraT3pAzzdOjgVawosuoyeY9R08/BngArtLftW5kSJtnqpmkTR4WnBEvFDH +zx6Y8sRW6npICpks/fDmmFpzE1wQG6Cqpsj5P/SnHqBBT86SpjhfdpfdZl+fXYJk +OAOsFdulscgiswXy5QpksI09fkCDdhVb9EL2lYUjjhZwLplzMk3jKA394J+yGx/t +kNGPhgKms15DOsrkZJigYJoM2cO6KpHQt6jSeVGIvOR3FWrQr/R27LowkHTj0R3e +QB8x3uL4IIfMisovVLheojJg0yfZFtWJN946M/E5JrWcTKa3yxJmGVCF6wvkExR1 +cGoz7tE8hfneHdrcSklKkKuBBgDkuHZnWgaF2/9muwtrvirnkCyXYCNyPJuh14tZ +FD5Ch3ya6MHonCIt6IZBt6kx2H7wpOAt8lS0C6K+pk/WpSLN18eiNkNwqvuy7USG +Xr8WNy1op+cM4LYE4+xlMNwVEqQ55TUEemBEnIgOaAZp/rX6lQEgnuRS/9A/AJVU +88JxDJL7ch111tpbyaF1vwNPIr5QyRX/WOG0qoFR9zrbscSp6fRoaCiTQlIEE6rC +99yGoaGiwg/4rVRH1ACQiRscYqEGAPXcFrHRUJc9iETM70IjKU3pQMxnlo3BNSrU +cHtBLKV0Mi55MQ5TKh/INYLSXUjV6f+ii/2cTKdCfU61Ex0bOx8y4X7IKaIaWjn+ +ALvs5g7sTSbDuvt+RQV+rSzZRmuFXyyc+7Y8w+F+VUPcJm8rLRdJH9UwFceBqTw4 +H3PJt+OgAVEa0qXI+csM8aBM3RmIows3N5u/NNUOEFkwRVjBnZ/q2tYDXjLFDFbI +DxvArQQr+UUO9bm4YUGd99b7oO017wYAqU0lUNrl+K/FNQo6XZDoWEXBCM//K+vA +E0PGtj+DZP/TFQ2hIlfZPEbHLIXc8Yq2qDz83z0rYd36LT1y4OvKlNQ9nppxzAxM +Svl06NZ7PUx2/Bql2loLQWfLPC/uY4teb167rx+TACfD/kOb8EXqyeYkjJNZeEWr +RPgfCi9yPXKTnaT1QdNOlf3ijPz/MbogZHk4wosC3PcFO1/tcABv79ovZJIBsLKR +EQOhjT+YaV53O+e6DVU1ArccaExL0cFl3+SJAbYEGAEKACAWIQTpXI0CYrueC+0C +KA5roXFu+wmdsgUCZipnqQIbDAAKCRBroXFu+wmdsm7JC/97EXNdkjPnJw7xIj+U +27NSisRKHtDKKlPJMPJ94Id1SKOwGECyTDUe3+51DvtyOVkmxHdOLal9QjdhCybI +SGJPklO2o+Af7WPmPdERcQZqLcxyGZJQe1s3zZ5MI/GNeu9Kci/bN0SITAeL1VOJ +V1BMgIhUVRtxITF95ypU05xi5BCrn2CmWnNWS1S3VthUY2Sp2DPO4KTlgH78gehP +VGUw7KrfNzsFiOEyzPDrwXpZe5Y9quwGnB8MFcapw59juN5bUZuJWPDEAbVozjDb +QCxEqhtXFZwsy+p8VWd8XfzyQ/Z6GDiuxb156FhRPbYfvqxrJVWqRSV5S3Om3QyG +zqufhduxDetIlj9/P37AuxZxxOFxtFb/lTN/qhs4vaWJ3NIiK414ok/DHi4WYm6R +cS+DRuARWktZ9KNlmyJ9/xDCgv3UT/x5QtXrwBeX97vOvC7EzVfbtlCc7TuA3Ay7 +h+EjflnLw/Nw84iZQKy1WTXN/kLEzZYK1ZTMNTIaE3Ps1Bo= +=dwTJ +-----END PGP PRIVATE KEY BLOCK----- From 4eeac1f512351fa4e4bf93851cdd9e5c5956fe6a Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 15:59:24 +0100 Subject: [PATCH 41/78] bootstrap: get database passwords from command line --- bootstrap/selfconfig | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 8e6c71428..98c933a89 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -5,6 +5,12 @@ use Carp qw(croak); use Getopt::Long::Descriptive; use Path::Tiny; +my ($opt, $usage) = describe_options( + '%c %o', + [ 'authuser-pw=s', "password for auth user", { required => 1 } ], + [ 'moduser-pw=s', "password for mod user", { required => 1 } ], +); + if (-e '/tmp/plenv-tarball.tar.bz2') { chdir("/home/pause") or die "can't chdir to ~pause: $!"; @@ -76,6 +82,9 @@ run_cmd(qw(git init --initial-branch master )); # 6BA1716EFB099DB2. -- rjbs, 2024-04-25 run_cmd(qw( gpg --import --armor /home/pause/pause/bootstrap/test-key.txt )); +my $authuser_pw = $opt->authuser_pw; +my $moduser_pw = $opt->moduser_pw; + path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); use strict; package PAUSE; @@ -84,10 +93,13 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); our $Config; $Config->{AUTHEN_BACKUP_DIR} = "/home/pause/db-backup"; - $Config->{AUTHEN_DATA_SOURCE_USER} = "pause"; - $Config->{AUTHEN_DATA_SOURCE_PW} = "pausepassword"; - $Config->{MOD_DATA_SOURCE_USER} = "pause"; - $Config->{MOD_DATA_SOURCE_PW} = "pausepassword"; + + $Config->{AUTHEN_DATA_SOURCE_USER} = "authuser"; + $Config->{AUTHEN_DATA_SOURCE_PW} = "\Q$authuser_pw"; + + $Config->{MOD_DATA_SOURCE_USER} = "moduser"; + $Config->{MOD_DATA_SOURCE_PW} = "\Q$moduser_pw"; + $Config->{MAIL_MAILER} = ["testfile"]; $Config->{RUNDATA} = "/tmp/pause_1999"; From 5268fdc445806d9153799e0093a42f847c995ab2 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 16:27:08 +0100 Subject: [PATCH 42/78] bootstrap: fix how pause user is put into config --- bootstrap/selfconfig | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig index 98c933a89..0944ef54b 100755 --- a/bootstrap/selfconfig +++ b/bootstrap/selfconfig @@ -82,10 +82,7 @@ run_cmd(qw(git init --initial-branch master )); # 6BA1716EFB099DB2. -- rjbs, 2024-04-25 run_cmd(qw( gpg --import --armor /home/pause/pause/bootstrap/test-key.txt )); -my $authuser_pw = $opt->authuser_pw; -my $moduser_pw = $opt->moduser_pw; - -path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); +my $config_file_contents = <<~'END'; use strict; package PAUSE; @@ -95,10 +92,10 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $Config->{AUTHEN_BACKUP_DIR} = "/home/pause/db-backup"; $Config->{AUTHEN_DATA_SOURCE_USER} = "authuser"; - $Config->{AUTHEN_DATA_SOURCE_PW} = "\Q$authuser_pw"; + $Config->{AUTHEN_DATA_SOURCE_PW} = "%%AUTHUSER_PW%%"; $Config->{MOD_DATA_SOURCE_USER} = "moduser"; - $Config->{MOD_DATA_SOURCE_PW} = "\Q$moduser_pw"; + $Config->{MOD_DATA_SOURCE_PW} = "%%MODUSER_PW%%"; $Config->{MAIL_MAILER} = ["testfile"]; $Config->{RUNDATA} = "/tmp/pause_1999"; @@ -122,6 +119,10 @@ path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew(<<~'END'); $Config->{TMP} = "/home/pause/tmp/"; END +$config_file_contents =~ s/%%AUTHUSER_PW%%/$opt->authuser_pw/e; +$config_file_contents =~ s/%%MODUSER_PW%%/$opt->moduser_pw/e; +path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew($config_file_contents); + sub run_cmd (@args) { system {$args[0]} @args; From 6b69a5b59dc59e19bceff6daa0685828063cd94d Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 18:20:22 +0100 Subject: [PATCH 43/78] bootstrap/import-mod-db: import both database dumps --- bootstrap/import-mod-db | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/bootstrap/import-mod-db b/bootstrap/import-mod-db index c736be831..d642c3419 100755 --- a/bootstrap/import-mod-db +++ b/bootstrap/import-mod-db @@ -4,8 +4,15 @@ if [ $UID != "0" ]; then exit 1; fi -rsync -vaP pause.perl.org::pausedata/moddump.current.bz2 . -bunzip2 moddump.current.bz2 +if [ ! -e "moddump.current.bz2" -o ! -e "authen_pausedump.current.bz2" ]; then + # The code used to do this: + # rsync -vaP pause.perl.org::pausedata/moddump.current.bz2 . + # + # ...but this only gets the moddump, and really we are going to need authen + # also. + echo "both moddump.current.bz2 and authen_pausedump.current.bz2 must be in cwd" >&2 + exit 1; +fi touch /etc/PAUSE.CLOSED @@ -15,10 +22,18 @@ sudo -u pause rsync --progress -av pause.perl.org::PAUSE/ /home/pause/pub/PAUSE/ echo 'SET GLOBAL innodb_flush_log_at_trx_commit=2' | mysql +bunzip2 moddump.current.bz2 + pv moddump.current | \ perl -ple 's/^CHANGE MASTER.*//' | \ sudo mysql mod +bunzip2 authen_pausedump.current.bz2 + +pv authen_pause.current | \ + perl -ple 's/^CHANGE MASTER.*//' | \ + sudo mysql authen + echo 'SET GLOBAL innodb_flush_log_at_trx_commit=1' | mysql systemctl start paused From 227298a68953820a2ac0d8d3372cb59fcf4155a0 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 18:46:04 +0100 Subject: [PATCH 44/78] bootstrap/import-pause-data: rename, program is no longer just db --- bootstrap/{import-mod-db => import-pause-data} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bootstrap/{import-mod-db => import-pause-data} (100%) diff --git a/bootstrap/import-mod-db b/bootstrap/import-pause-data similarity index 100% rename from bootstrap/import-mod-db rename to bootstrap/import-pause-data From 568265a9355f12456e6baf9e07ed9c0a66b0d89d Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Thu, 25 Apr 2024 18:51:56 +0100 Subject: [PATCH 45/78] bootstrap: consolidate old "boxo" program into pause.git --- bootstrap/lib/Dobby/Client.pm | 315 +++++++++++++++++++++ bootstrap/mkpause | 295 +++++++++++++++++++ bootstrap/{selfconfig => selfconfig-pause} | 0 bootstrap/selfconfig-root | 274 ++++++++++++++++++ 4 files changed, 884 insertions(+) create mode 100644 bootstrap/lib/Dobby/Client.pm create mode 100755 bootstrap/mkpause rename bootstrap/{selfconfig => selfconfig-pause} (100%) create mode 100755 bootstrap/selfconfig-root diff --git a/bootstrap/lib/Dobby/Client.pm b/bootstrap/lib/Dobby/Client.pm new file mode 100644 index 000000000..4998e7f9e --- /dev/null +++ b/bootstrap/lib/Dobby/Client.pm @@ -0,0 +1,315 @@ +use v5.32.0; +use warnings; + +package Dobby::Client; + +use experimental 'signatures'; +use parent 'IO::Async::Notifier'; + +use Carp (); +use Future::AsyncAwait; +use Future::Utils qw(repeat); +use IO::Async::Loop; +use JSON::MaybeXS; +use Net::Async::HTTP; + +sub configure ($self, %param) { + my @missing; + + KEY: for my $key (qw( bearer_token )) { + unless (defined $param{$key}) { + push @missing, $key; + next KEY; + } + + $self->{"__dobby_$key"} = $param{$key}; + } + + if (@missing) { + Carp::confess("missing required Dobby::Client parameters: @missing"); + } + + return; +} + +sub api_base { + return 'https://api.digitalocean.com/v2'; +} + +sub bearer_token { $_[0]{__dobby_bearer_token} } + +sub http ($self) { + return $self->{__dobby_http} //= do { + my $http = Net::Async::HTTP->new( + user_agent => 'Dobby/0', + ); + + $self->loop->add($http); + + $http; + }; +} + +async sub json_get ($self, $path) { + my $res = await $self->http->do_request( + method => 'GET', + uri => $self->api_base . $path, + headers => { + 'Authorization' => "Bearer " . $self->bearer_token, + }, + ); + + unless ($res->is_success) { + die "error getting $path at DigitalOcean: " . $res->as_string; + } + + my $json = $res->decoded_content(charset => undef); + decode_json($json); +} + +async sub json_get_pages_of ($self, $path, $key) { + my $url = $self->api_base . $path; + + my @items; + + while ($url) { + my $res = await $self->http->do_request( + method => 'GET', + uri => $url, + headers => { + 'Authorization' => "Bearer " . $self->bearer_token, + }, + ); + + unless ($res->is_success) { + die "error getting $path at DigitalOcean: " . $res->as_string; + } + + my $json = $res->decoded_content(charset => undef); + my $data = decode_json($json); + + die "no entry for $key in returned page" + unless exists $data->{$key}; + + push @items, $data->{$key}->@*; + $url = $data->{links}{pages}{next}; + } + + return \@items; +} + +async sub _json_req_with_body ($self, $method, $path, $payload) { + my $res = await $self->http->do_request( + method => $method, + uri => $self->api_base . $path, + headers => { + 'Authorization' => "Bearer " . $self->bearer_token, + }, + + content_type => 'application/json', + content => encode_json($payload), + ); + + unless ($res->is_success) { + die "error making $method to $path at DigitalOcean: " . $res->as_string; + } + + my $json = $res->decoded_content(charset => undef); + decode_json($json); +} + +async sub json_post ($self, $path, $payload) { + await $self->_json_req_with_body('POST', $path, $payload); +} + +async sub json_put ($self, $path, $payload) { + await $self->_json_req_with_body('PUT', $path, $payload); +} + +async sub delete_url ($self, $path) { + my $res = await $self->http->do_request( + method => 'DELETE', + uri => $self->api_base . $path, + headers => { + 'Authorization' => "Bearer " . $self->bearer_token, + }, + ); + + unless ($res->is_success) { + die "error deleting resource at $path in DigitalOcean: " . $res->as_string; + } + + return; +} + +async sub create_droplet ($self, $arg) { + state @required_keys = qw( name region size tags image ssh_keys ); + + my @missing; + KEY: for my $key (@required_keys) { + unless (defined $arg->{$key}) { + push @missing, $key; + next KEY; + } + } + + if (@missing) { + Carp::confess("missing required Dobby::Client parameters: @missing"); + } + + my $create_res = await $self->json_post( + "/droplets", + { + $arg->%{ @required_keys }, + }, + ); + + my $droplet = $create_res->{droplet}; + + unless ($droplet) { + Carp::confess("Error creating Droplet."); + } + + my $action_id = $create_res->{links}{actions}[0]{id}; + + unless (defined $action_id) { + Carp::confess( + "no action id from droplet action: " . encode_json($create_res) + ); + } + + my $waited = await $self->_do_action_status_f("/actions/$action_id"); + + return $droplet; +} + +async sub take_droplet_action ($self, $droplet_id, $action, $payload = {}) { + my $action_res = await $self->json_post("/droplets/$droplet_id/actions", { + %$payload, + type => $action, + }); + + my $action_id = $action_res->{action}{id}; + + unless (defined $action_id) { + Carp::confess( + "no action id from droplet action: " . encode_json($action_res) + ); + } + + await $self->_do_action_status_f("/droplets/$droplet_id/actions/$action_id"); + + return; +} + +async sub destroy_droplet ($self, $droplet_id) { + my $delete_res = await $self->delete_url("/droplets/$droplet_id"); + return; +} + +async sub _do_action_status_f ($self, $action_url) { + TRY: while (1) { + my $action = await $self->json_get($action_url); + my $status = $action->{action}{status}; + + # ugh, DO is now sometimes returning empty string in the status field + # -- michael, 2021-04-16 + $status = 'completed' if ! $status && $action->{action}{completed_at}; + + if ($status eq 'in-progress') { + await $self->loop->delay_future(after => 5); + next TRY; + } + + if ($status eq 'completed') { + return; + } + + if ($status eq 'errored') { + Carp::confess("action $action_url failed: " . encode_json($action->{action})); + } + + Carp::confess("action $action_url in unknown state: $status"); + } +} + +async sub _get_droplets ($self, $arg = {}) { + my $path = '/droplets?per_page=200'; + $path .= "&tag_name=$arg->{tag}" if $arg->{tag}; + + my $droplets_data = await $self->json_get($path); + + # TODO Obviously, this should lazily fetch etc. + if ($droplets_data->{links}{pages}{forward_links}) { + Carp::cluck("Single-page fetch did not find all droplets!"); + } + + unless ($droplets_data->{droplets}) { + Carp::cluck( + "getting /droplets didn't supply droplets: " . encode_json($droplets_data) + ); + } + + return $droplets_data->{droplets}->@*; +} + +async sub get_all_droplets ($self) { + await $self->_get_droplets; +} + +async sub get_droplets_with_tag ($self, $tag) { + await $self->_get_droplets({ tag => $tag }); +} + +async sub add_droplet_to_project ($self, $droplet_id, $project_id) { + my $path = "/projects/$project_id/resources"; + + await $self->json_post($path, { + resources => [ "do:droplet:$droplet_id" ], + }); +} + +async sub get_all_domain_records_for_domain ($self, $domain) { + my $path = '/domains/' . $domain . '/records'; + + # TODO Obviously, this should lazily fetch etc. + my $record_res = await $self->json_get("$path?per_page=200"); + return unless $record_res->{domain_records}; + return $record_res->{domain_records}->@*; +} + +async sub remove_domain_records_for_ip ($self, $domain, $ip) { + my $path = '/domains/' . $domain . '/records'; + + my @records = await $self->get_all_domain_records_for_domain($domain); + my @to_delete = grep {; $_->{data} eq $ip } @records; + my @deletions = map {; $self->delete_url("$path/$_->{id}") } @to_delete; + + return await Future->wait_all(@deletions); +} + +async sub point_domain_record_at_ip ($self, $domain, $name, $ip) { + my $path = '/domains/' . $domain . '/records'; + + my @records = await $self->get_all_domain_records_for_domain($domain); + + my (@existing) = grep {; $_->{name} eq $name && $_->{type} eq 'A' } @records; + + if (@existing) { + my @to_update = map {; $self->json_put("$path/$_->{id}", { data => $ip }) } + @existing; + await Future->wait_all(@to_update); + return; + } + + await $self->json_post($path, { + type => 'A', + name => $name, + data => $ip, + ttl => 30, + }); + + return; +} + +1; diff --git a/bootstrap/mkpause b/bootstrap/mkpause new file mode 100755 index 000000000..1fa30b21b --- /dev/null +++ b/bootstrap/mkpause @@ -0,0 +1,295 @@ +#!/usr/bin/env perl +use v5.36.0; + +use lib 'lib'; + +use Dobby::Client; +use Future::AsyncAwait; +use Getopt::Long::Descriptive; +use IO::Async::Loop; +use Log::Dispatchouli; + +my ($opt, $usage) = describe_options( + '%c %o', + [ 'username|u=s', "your username; defaults to $ENV{USER}", + { default => $ENV{USER} // die 'no USER env var!' } ], + + # For serious business, we like c-16. -- rjbs, 2024-04-19 + [ 'size=s', "slug for Digital Ocean droplet", { default => 'g-4vcpu-16gb' } ], + [ 'box-ident|i=s', "identifying part of box name; defaults to --username" ], + [], + [ 'plenv-file|P=s', "a tar.bz2 file to use for plenv (it's a shortcut)" ], + [], + [ 'certbot-staging|C', 'use the staging version of certbot' ], + + [], + [ 'mode', 'hidden' => { + default => 'create', + one_of => [ + [ 'create', 'create the box if it does not exist' ], + [ 'destroy', 'destroy the box if it does exist' ], + ], + } + ], +); + +my $Logger = Log::Dispatchouli->new({ + facility => undef, + ident => 'unpause-boxo', + log_pid => 0, + to_stdout => 1, +}); + +if ($opt->plenv_file) { + die "plenv file does not exist" unless -e $opt->plenv_file; + die "plenv file can't be read" unless -r $opt->plenv_file; + + die "plenv file should be a .tar.bz2 file" + unless $opt->plenv_file =~ /\.tar\.bz2\z/; +} + +my $loop = IO::Async::Loop->new; + +my $TOKEN = $ENV{DO_TOKEN} // die "no token, no milk\n"; + +my $dobby = Dobby::Client->new( + bearer_token => $TOKEN, +); + +$loop->add($dobby); + +my $domain = "fastmail.dev"; +my $username = $opt->username; +my $boxname = ($opt->box_ident // $username) . ".unpause"; +my $project_id = q{62113225-81be-4538-8408-f42f54f6c93f}; # PAUSE 2024 + +my $todo = __PACKAGE__->can("do_" . $opt->mode); + +# This "can't happen". -- rjbs, 2024-03-23 +die "WTF: unknown mode of operation request" unless $todo; + +await $todo->(); + +#---( cut here )--- + +my sub ip_addr_for ($droplet) { + my ($ip_addr) = map { $_->{ip_address} } grep { $_->{type} eq 'public'} + $droplet->{networks}{v4}->@*; + + return $ip_addr; +} + +async sub do_create { + { + my @droplets = await $dobby->get_droplets_with_tag('unpause'); + my ($droplet) = grep {; $_->{name} eq $boxname } @droplets; + + if ($droplet) { + my ($net) = grep {; + $_->{type} eq 'public' + } $droplet->{networks}{v4}->@*; + + my $extra = $net ? " at root\@$net->{ip_address}" : ""; + + die "box already exists$extra\n"; + } + } + + my @key_ids; + { + my %want_key = map {; $_ => 1 } qw( matthew rjbs ); + my $keys = await $dobby->json_get_pages_of("/account/keys", 'ssh_keys'); + + my (@keys) = grep {; $want_key{$_->{name}} } @$keys; + + unless (@keys) { + die "can't find ssh keys to use!\n"; + } + + @key_ids = map {; $_->{id} } @keys; + } + + my $image = 'debian-12-x64'; + my $region = 'nyc3'; + my $size = $opt->size; + + my %droplet_create_args = ( + name => $boxname, + image => $image, + region => $region, + size => $size, + ssh_keys => \@key_ids, + tags => [ 'unpause' ], + ); + + $Logger->log([ "Creating droplet: %s", \%droplet_create_args ]); + + my $droplet = await $dobby->create_droplet(\%droplet_create_args); + + unless ($droplet) { + die "There was an error creating the box. Try again.\n"; + } + + # At this point, the box exists, but isn't quite up. The above result, for + # example, has no networks entry. We need to re-get it. + $Logger->log([ "Created droplet %i, now waiting for network...", $droplet->{id} ]); + + # We delay this because a completed droplet sometimes does not show up in GET + # /droplets immediately, which causes annoying problems. Waiting is a + # silly fix, but seems to work, and it's not like box creation is + # lightning-fast anyway. + await $loop->delay_future(after => 5); + + { + my $payload = await $dobby->json_get("/droplets/$droplet->{id}"); + $droplet = $payload->{droplet}; + } + + unless ($droplet) { + die "Box was created, but now I can't find it! Check the DigitalOcean console and maybe try again.\n"; + } + + my $ip_addr = ip_addr_for($droplet); + + $Logger->log([ "Droplet is now up on %s...", $ip_addr ]); + + await $dobby->add_droplet_to_project($droplet->{id}, $project_id); + + $Logger->log("updating DNS names for $boxname"); + + await $dobby->point_domain_record_at_ip( + $domain, + "$boxname", + $ip_addr, + ); + + $Logger->log("Waiting for ssh to become available..."); + + my $ssh_up = await wait_for_port($ip_addr, 22); + + unless ($ssh_up) { + $Logger->log("The droplet was created, but ssh didn't come up. Your turn!"); + exit 1; + } + + $Logger->log("ssh is now available to $boxname.$domain ($ip_addr)"); + + if ($opt->plenv_file) { + $Logger->log("Copying plenv install tarball to destination..."); + + system( + qw( + scp + -o UserKnownHostsFile=/dev/null + -o UpdateHostKeys=no + -o StrictHostKeyChecking=no + ), + $opt->plenv_file, + + "root\@$ip_addr:/tmp/plenv-tarball.tar.bz2", + ); + } + + + $Logger->log("Now turning the box into a PAUSE server..."); + + system( + qw( + scp + -o UserKnownHostsFile=/dev/null + -o UpdateHostKeys=no + -o StrictHostKeyChecking=no + + selfconfig-root + ), + "root\@$ip_addr:", + ); + + system( + qw( + ssh + -o UserKnownHostsFile=/dev/null + -o UpdateHostKeys=no + -o StrictHostKeyChecking=no + + -l root + ), + $ip_addr, + qw( perl selfconfig-root ), + '--host', "$boxname.$domain", + '--user', $username, + '--pass', $username, # XXX: Do something better later. + ($opt->certbot_staging ? '--certbot-staging' : ()), + ); + + $Logger->log(<<~EOF); + + Done! If all went well, pause should be at: + + ssh root\@$ip_addr + ssh root\@$boxname.$domain + + https://$boxname.$domain + EOF +} + +async sub do_destroy { + my $droplet; + { + my @droplets = await $dobby->get_droplets_with_tag('unpause'); + ($droplet) = grep {; $_->{name} eq $boxname } @droplets; + } + + unless ($droplet) { + die "The box $boxname does not exist, and so cannot be destroyed.\n"; + } + + my $ip_addr = ip_addr_for($droplet); + + await $dobby->remove_domain_records_for_ip($domain, $ip_addr); + + $Logger->log([ "Destroying droplet: %s (%s)", $droplet->{id}, $droplet->{name} ]); + + await $dobby->destroy_droplet($droplet->{id}); + + $Logger->log([ "Destroyed droplet: %s", $droplet->{id} ]); +} + +async sub wait_for_port ($ip_addr, $port) { + my $max_tries = 20; + TRY: for my $try (1..$max_tries) { + my $socket; + eval { + $socket = await $loop->connect(addr => { + family => 'inet', + socktype => 'stream', + port => 22, + ip => $ip_addr, + }); + }; + + if ($socket) { + # We didn't need the connection, just to know it worked! + return 1; + } + + my $error = $@; + if ($error !~ /Connection refused/) { + $Logger->log([ + "weird error connecting to %s:22: %s", + $ip_addr, + $error, + ]); + } + + $Logger->log([ + "ssh on %s is not up, maybe wait and try again; %s tries remain", + $ip_addr, + $max_tries - $try, + ]); + + await $loop->delay_future(after => 1); + } + + return; +} diff --git a/bootstrap/selfconfig b/bootstrap/selfconfig-pause similarity index 100% rename from bootstrap/selfconfig rename to bootstrap/selfconfig-pause diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root new file mode 100755 index 000000000..88da7f51b --- /dev/null +++ b/bootstrap/selfconfig-root @@ -0,0 +1,274 @@ +#!/usr/bin/perl +use v5.36.0; +use warnings; + +use Carp qw(croak); + +# If we don't have a term debconf gets angry +$ENV{TERM} //= 'xterm'; + +# On DigitalOcean, the journal won't be journaling at startup. Why? Well, I +# want to swear and say "because systemd!" but there seems to be an interesting +# reason, related to the machine id being baked into the image and then not +# matching that on the new cloud instance. I don't quite follow it. +# +# References: +# * https://unix.stackexchange.com/a/538881 +# * https://serverfault.com/a/1058260 +# +# Since we won't (??) be using DO for this in real work, I'm not trying to +# really fix it, I just want it logging before we start doing work. This will +# do the trick: +run_cmd(qw( systemctl restart systemd-journald.service )); + +# Don't run apt-get update if apt is already busy. We need to wait or we'll +# fail to update. Also wait for /var/lib/dpkg/lock-frontend +for my $try (1..30) { + system( + "fuser /var/lib/apt/lists/lock >/dev/null 2>/dev/null" + ); + + my $exit = $? >> 8; + last if $exit; + + warn "apt running, waiting 1s, try $try/30\n"; + + sleep 1; +} + +run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 update)); + +# Install system deps: +# +# Note that rjbs has been somewhat obnoxiously clever, below. Here, we install +# libpath-tiny-perl. That's Path::Tiny. Later in this very program, we will +# load and use this module. What a yutz. -- rjbs, 2024-03-23 +# +# Same goes for libgetopt-long-descriptive-perl. Yutz and proud. +# -- rjbs, 2024-04-05 +my @required_debs = qw( + build-essential + certbot + default-libmysqlclient-dev + git + libdb-dev + libexpat1-dev + libgetopt-long-descriptive-perl + libpath-tiny-perl + libssl-dev + mariadb-server + nginx + python3-certbot-nginx + unzip + zlib1g-dev +); + +run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y), @required_debs); + +require Getopt::Long::Descriptive; + +my ($opt, $usage) = Getopt::Long::Descriptive::describe_options( + '%c %o', + [ "host=s", "the hostname being used for this install", { required => 1 } ], + [ "user=s", "username for PAUSE admin to create", { required => 1 } ], + [ "pass=s", "password for PAUSE admin to create", { required => 1 } ], + [], + [ 'certbot-staging|C', 'use the staging version of certbot' ], +); + +my $hostname = $opt->host; +my $admin_user = uc $opt->user; +my $admin_pass = $opt->pass; + +# The --comment is here to suppress prompting for name, confirmation, etc. +run_cmd(qw(adduser pause --disabled-password --comment), 'PAUSE User'); +run_cmd(qw(adduser unsafe --disabled-password --comment), 'PAUSE Unsafe'); + +run_cmd(qw( + sudo -u pause git clone -b unpause https://git@github.com/rjbs/pause/ /home/pause/pause +)); + +# set up mysql databases and our pause user +run_cmd(qw(mysqladmin CREATE mod)); +run_sh('mysql mod < ~pause/pause/doc/mod.schema.txt'); + +run_cmd(qw(mysqladmin CREATE authen_pause)); +run_sh('mysql -u root authen_pause < ~pause/pause/doc/authen_pause.schema.txt'); + +run_cmd(qw(mysql mod -e), "INSERT INTO users (userid) VALUES ('$admin_user')"); + +my $crypted_pass = crypt $admin_pass, chr(rand(26)+97) . chr(rand(26)+97); + +run_cmd( + qw(mysql authen_pause -e), + "INSERT INTO usertable (user, password) VALUES ('$admin_user', '$crypted_pass')", +); + +run_cmd( + qw(mysql authen_pause -e), + "INSERT INTO grouptable (user, ugroup) VALUES ('$admin_user', 'admin')", +); + +my %db_password_for = ( + authuser => undef, + moduser => undef, +); + +{ + my sub rand_pw { + # Generates strings kinda like this one: b9l12-r5y9s-uc609-zey9q-61vjd + my @chars = (0..9, 'a' .. 'z'); + my $pw = join q{-}, + map {; join q{}, map {; $chars[ rand @chars ] } (1..5) } + (1..5); + + return $pw; + } + + for my $user (sort keys %db_password_for) { + $db_password_for{$user} = rand_pw(); + + run_cmd( + qw(mysql -e), + qq{CREATE USER $user IDENTIFIED BY '$db_password_for{$user}'}, + ); + } +} + +run_cmd( + qw(mysql -e), + q{GRANT DELETE, INDEX, INSERT, SELECT, UPDATE, LOCK TABLES ON `mod`.* TO 'moduser'@'%';}, +); + +run_cmd( + qw(mysql -e), + q{GRANT DELETE, INDEX, INSERT, SELECT, UPDATE, LOCK TABLES ON `authen_pause`.* TO 'authuser'@'%';}, +); + +run_cmd( + qw(mysql -e), + q{GRANT BINLOG MONITOR, RELOAD ON *.* TO 'moduser'@'%';}, +); + +run_cmd( + qw(mysql -e), + q{GRANT BINLOG MONITOR, RELOAD ON *.* TO 'authuser'@'%';}, +); + +my $nginx_config = <<~"END"; +# Set up nginx conf +upstream pause { + server 127.0.0.1:5000; +} + +server { + listen 80 default_server; + + location / { + proxy_pass http://pause; + proxy_set_header X-Forwarded-Host \$host; + proxy_set_header X-Forwarded-Server \$host; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto \$scheme; + + proxy_pass_request_headers on; + proxy_no_cache \$cookie_nocache \$arg_nocache\$arg_comment; + proxy_no_cache \$http_pragma \$http_authorization; + proxy_cache_bypass \$cookie_nocache \$arg_nocache \$arg_comment; + proxy_cache_bypass \$http_pragma \$http_authorization; + proxy_pass_header Authorization; + } + + server_name $hostname; +} +END + +require Path::Tiny; +Path::Tiny::path("/etc/nginx/sites-available/$hostname")->spew($nginx_config); + +unlink('/etc/nginx/sites-enabled/default'); +symlink("/etc/nginx/sites-available/$hostname", "/etc/nginx/sites-enabled/$hostname") + or die "can't symlink nginx conf: $!"; + +# Install ssl cert +run_cmd( + qw(sudo certbot --nginx -d), + $hostname, + qw(--agree-tos -n --email pause@pause.perl.org), + + # This will use the staging server, which can be used to make lots more + # certificates that usual, but they aren't trusted. + ($opt->certbot_staging + ? ( qw( --server https://acme-staging-v02.api.letsencrypt.org/directory ) ) + : ()), +); + +Path::Tiny::path("/etc/rsyncd.conf")->spew(<<~'END'); + # cat rsyncd.conf + max connections = 12 + log file = /var/log/rsyncd + pid file = /var/run/PAUSE-rsyncd.pid + transfer logging = true + use chroot = true + timeout = 600 + + [PAUSE] + path = /home/pause/pub/PAUSE + + [authors] + path = /home/pause/pub/PAUSE/authors + + [modules] + path = /home/pause/pub/PAUSE/modules + + [scripts] + path = /home/pause/pub/PAUSE/scripts/new + + [pausedata] + path = /home/pause/pub/PAUSE/PAUSE-data + + [pausecode] + path = /home/pause/pub/PAUSE/PAUSE-code + + [pausegit] + path = /home/pause/pub/PAUSE/PAUSE-git + END + +run_cmd( + qw( sudo -u pause ), + "/home/pause/pause/bootstrap/selfconfig-pause", + "--authuser-pw", $db_password_for{authuser}, + "--moduser-pw", $db_password_for{moduser}, +); + +Path::Tiny::path("/home/pause/pause/cron/CRONTAB.ROOT")->copy("/etc/cron.d/pause"); + +for my $service (qw( paused pause-web )) { + Path::Tiny::path("/home/pause/pause/services/$service.service") + ->copy("/etc/systemd/system/$service.service"); + + run_cmd(qw( systemctl enable ), $service ); + run_cmd(qw( systemctl start ), $service ); + run_cmd(qw( systemctl status ), $service ); +} + +for my $service (qw( rsync )) { + run_cmd(qw( systemctl enable ), $service ); + run_cmd(qw( systemctl start ), $service ); + run_cmd(qw( systemctl status ), $service ); +} + +## SUBROUTINES GO DOWN BELOW +## THAT'S WHY THEY'RE CALLED SUBROUTINES + +sub run_cmd (@args) { + system {$args[0]} @args; + + croak "failed to run $args[0]" if $?; +} + +sub run_sh ($str) { + system $str; + + croak "failed to run shell command" if $?; +} From 3078a187ed87cdf61dd0f3d212bd7a8eed9c5334 Mon Sep 17 00:00:00 2001 From: Matthew Horsfall Date: Fri, 26 Apr 2024 09:17:01 +0100 Subject: [PATCH 46/78] bootstrap: add an --enable-mail option to enable mail --- bootstrap/mkpause | 28 +++++++++++++++ bootstrap/selfconfig-root | 73 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/bootstrap/mkpause b/bootstrap/mkpause index 1fa30b21b..596ab19d6 100755 --- a/bootstrap/mkpause +++ b/bootstrap/mkpause @@ -21,6 +21,11 @@ my ($opt, $usage) = describe_options( [ 'plenv-file|P=s', "a tar.bz2 file to use for plenv (it's a shortcut)" ], [], [ 'certbot-staging|C', 'use the staging version of certbot' ], + [ 'enable-mail|m', "set up postfix config for outbound mail" ], + [ 'relay-host=s', "relay host for smtp" ], + [ 'relay-port=s', "relay port for smtp" ], + [ 'relay-username=s', "relay sasl username for smtp" ], + [ 'relay-password=s', "relay sasl password for smtp" ], [], [ 'mode', 'hidden' => { @@ -33,6 +38,20 @@ my ($opt, $usage) = describe_options( ], ); +my @mopts = qw(relay_host relay_port relay_username relay_password); + +if ($opt->enable_mail) { + my @m; + + for my $setting (@mopts) { + push @m, $setting unless $opt->$setting; + } + + $_ =~ s/_/-/g for @m; + + die "--enable-mail requires @m\n" if @m; +} + my $Logger = Log::Dispatchouli->new({ facility => undef, ident => 'unpause-boxo', @@ -174,6 +193,8 @@ async sub do_create { $Logger->log("ssh is now available to $boxname.$domain ($ip_addr)"); + sleep 1; + if ($opt->plenv_file) { $Logger->log("Copying plenv install tarball to destination..."); @@ -205,6 +226,12 @@ async sub do_create { "root\@$ip_addr:", ); + my %mailopts; + + if ($opt->enable_mail) { + $mailopts{"--" . ($_ =~ s/_/-/gr)} = $opt->$_ for @mopts; + } + system( qw( ssh @@ -220,6 +247,7 @@ async sub do_create { '--user', $username, '--pass', $username, # XXX: Do something better later. ($opt->certbot_staging ? '--certbot-staging' : ()), + ($opt->enable_mail ? ('--enable-mail', %mailopts) : ()), ); $Logger->log(<<~EOF); diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 88da7f51b..b466f8434 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -61,6 +61,7 @@ my @required_debs = qw( python3-certbot-nginx unzip zlib1g-dev + libsasl2-modules ); run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y), @required_debs); @@ -74,8 +75,27 @@ my ($opt, $usage) = Getopt::Long::Descriptive::describe_options( [ "pass=s", "password for PAUSE admin to create", { required => 1 } ], [], [ 'certbot-staging|C', 'use the staging version of certbot' ], + [ "enable-mail|m", "enable working postfix config", ], + [ 'relay-host=s', "relay host for smtp" ], + [ 'relay-port=s', "relay port for smtp" ], + [ 'relay-username=s', "relay sasl username for smtp" ], + [ 'relay-password=s', "relay sasl password for smtp" ], ); +my @mopts = qw(relay_host relay_port relay_username relay_password); + +if ($opt->enable_mail) { + my @m; + + for my $setting (@mopts) { + push @m, $setting unless $opt->$setting; + } + + $_ =~ s/_/-/g for @m; + + die "--enable-mail requires @m\n" if @m; +} + my $hostname = $opt->host; my $admin_user = uc $opt->user; my $admin_pass = $opt->pass; @@ -243,6 +263,59 @@ run_cmd( Path::Tiny::path("/home/pause/pause/cron/CRONTAB.ROOT")->copy("/etc/cron.d/pause"); +if ($opt->enable_mail) { + system( + q{echo "postfix postfix/main_mailer_type string 'Internet with smarthost'" | debconf-set-selections} + ); + + croak "failed to update debconf for main_mailer_type" if $?; + + system( + qq{echo "postfix postfix/mailname string '$hostname'" | debconf-set-selections} + ); + + croak "failed to update debconf for mailname" if $?; + + run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y postfix)); + + my $cf = Path::Tiny::path("/home/pause/pause/etc/puppet/modules/pause/files/etc/postfix/main.cf-pause-us"); + + my $maincf = $cf->slurp_raw; + $maincf =~ s{daemon_directory = /usr/libexec/postfix}{daemon_directory = /usr/lib/postfix/sbin} + or warn "!!! Failed to replace daemon_directory !!!\n\n"; + $maincf =~ s{inet_interfaces = all}{inet_interfaces = localhost} + or warn "!!! Failed to replace inet_interfaces !!!\n\n"; + + my $relayhost = $opt->relay_host; + my $relayport = $opt->relay_port; + + $maincf .= <<~EOF; + relayhost = [$relayhost]:$relayport + + smtp_sasl_auth_enable = yes + smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd + smtp_sasl_security_options = + smtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crt + smtp_use_tls = yes + smtp_tls_wrappermode = yes + smtp_tls_security_level = encrypt + EOF + + Path::Tiny::path("/etc/postfix/main.cf")->spew_raw($maincf); + + my ($user, $pass) = ($opt->relay_username, $opt->relay_password); + + Path::Tiny::path("/etc/postfix/sasl_passwd")->spew_raw(<<~EOF); + [$relayhost]:$relayport $user:$pass + EOF + + run_cmd(qw(chmod 600 /etc/postfix/sasl_passwd)); + run_cmd(qw(postmap /etc/postfix/sasl_passwd)); + + run_cmd(qw( postfix stop )); + run_cmd(qw( postfix start )); +} + for my $service (qw( paused pause-web )) { Path::Tiny::path("/home/pause/pause/services/$service.service") ->copy("/etc/systemd/system/$service.service"); From c154bb524b1d950ec61f2d8f8638d3023aeeb941 Mon Sep 17 00:00:00 2001 From: Matthew Horsfall Date: Fri, 26 Apr 2024 05:49:19 -0400 Subject: [PATCH 47/78] bootstrap: Override CRONPATH to the new location in Config --- bootstrap/selfconfig-pause | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index 0944ef54b..9ffd5aefb 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -104,6 +104,8 @@ my $config_file_contents = <<~'END'; $Config->{CHECKSUMS_SIGNING_ARGS} = '--homedir /home/pause/pause-private/gnupg-pause-batch-signing-home --clearsign --default-key '; $Config->{CHECKSUMS_SIGNING_KEY} = '6BA1716EFB099DB2'; + $Config->{CRONPATH} = '/home/pause/pause/cron/'; + $Config->{FTPPUB} = '/home/pause/pub/PAUSE/'; $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; $Config->{INCOMING_LOC} = '/home/pause/incoming'; From c3866ab80c2674eac3582f4dc6ce96540898e882 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 10:41:06 +0100 Subject: [PATCH 48/78] bootstrap: replace -P scp-ing with curl-ing This lets us have one common dot-plenv file, and eliminates some back and forth during bootstrap. --- bootstrap/mkpause | 29 +++-------------------------- bootstrap/selfconfig-root | 7 +++++++ 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/bootstrap/mkpause b/bootstrap/mkpause index 596ab19d6..874efcffc 100755 --- a/bootstrap/mkpause +++ b/bootstrap/mkpause @@ -18,7 +18,7 @@ my ($opt, $usage) = describe_options( [ 'size=s', "slug for Digital Ocean droplet", { default => 'g-4vcpu-16gb' } ], [ 'box-ident|i=s', "identifying part of box name; defaults to --username" ], [], - [ 'plenv-file|P=s', "a tar.bz2 file to use for plenv (it's a shortcut)" ], + [ 'plenv-url|P=s', "URL to a tar.bz2 file to use for plenv (it's a shortcut)" ], [], [ 'certbot-staging|C', 'use the staging version of certbot' ], [ 'enable-mail|m', "set up postfix config for outbound mail" ], @@ -59,14 +59,6 @@ my $Logger = Log::Dispatchouli->new({ to_stdout => 1, }); -if ($opt->plenv_file) { - die "plenv file does not exist" unless -e $opt->plenv_file; - die "plenv file can't be read" unless -r $opt->plenv_file; - - die "plenv file should be a .tar.bz2 file" - unless $opt->plenv_file =~ /\.tar\.bz2\z/; -} - my $loop = IO::Async::Loop->new; my $TOKEN = $ENV{DO_TOKEN} // die "no token, no milk\n"; @@ -195,23 +187,6 @@ async sub do_create { sleep 1; - if ($opt->plenv_file) { - $Logger->log("Copying plenv install tarball to destination..."); - - system( - qw( - scp - -o UserKnownHostsFile=/dev/null - -o UpdateHostKeys=no - -o StrictHostKeyChecking=no - ), - $opt->plenv_file, - - "root\@$ip_addr:/tmp/plenv-tarball.tar.bz2", - ); - } - - $Logger->log("Now turning the box into a PAUSE server..."); system( @@ -248,6 +223,8 @@ async sub do_create { '--pass', $username, # XXX: Do something better later. ($opt->certbot_staging ? '--certbot-staging' : ()), ($opt->enable_mail ? ('--enable-mail', %mailopts) : ()), + + ($opt->plenv_url ? ('--plenv-url' => $opt->plenv_url) : ()), ); $Logger->log(<<~EOF); diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index b466f8434..1aff07092 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -74,7 +74,10 @@ my ($opt, $usage) = Getopt::Long::Descriptive::describe_options( [ "user=s", "username for PAUSE admin to create", { required => 1 } ], [ "pass=s", "password for PAUSE admin to create", { required => 1 } ], [], + [ 'plenv-url=s', "curl-able URL to a tar.bz2 archive of a .plenv" ], + [], [ 'certbot-staging|C', 'use the staging version of certbot' ], + [], [ "enable-mail|m", "enable working postfix config", ], [ 'relay-host=s', "relay host for smtp" ], [ 'relay-port=s', "relay port for smtp" ], @@ -104,6 +107,10 @@ my $admin_pass = $opt->pass; run_cmd(qw(adduser pause --disabled-password --comment), 'PAUSE User'); run_cmd(qw(adduser unsafe --disabled-password --comment), 'PAUSE Unsafe'); +if ($opt->plenv_url) { + run_cmd('curl', $opt->plenv_url, '--output', '/tmp/plenv-tarball.tar.bz2'); +} + run_cmd(qw( sudo -u pause git clone -b unpause https://git@github.com/rjbs/pause/ /home/pause/pause )); From 70a31cea6a496b5bea521416244f083705bea839 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 11:04:10 +0100 Subject: [PATCH 49/78] bootstrap: add brief explanatory comments to programs yet to come: a README --- bootstrap/import-pause-data | 8 ++++++++ bootstrap/mkpause | 11 +++++++++++ bootstrap/selfconfig-pause | 13 +++++++++++++ bootstrap/selfconfig-root | 14 ++++++++++++++ 4 files changed, 46 insertions(+) diff --git a/bootstrap/import-pause-data b/bootstrap/import-pause-data index d642c3419..598c347cd 100755 --- a/bootstrap/import-pause-data +++ b/bootstrap/import-pause-data @@ -1,4 +1,12 @@ #!/bin/bash + +# This program is meant to be run on a just-installed bootstrapped PAUSE host. +# It expects that you've gotten MySQL dumps from ../cron/mysql-dump.pl, +# preferably using --extended-insert, and put them in the current working +# directory. It will stop indexing, import all the files from PAUSE, and then +# also all the database content. Then it starts things back up. This can be +# used for testing operations on "real" data, but is also a useful step in +# transitioning to a new PAUSE install in production. if [ $UID != "0" ]; then echo "import-mod-db is meant to be run as root" >&2 exit 1; diff --git a/bootstrap/mkpause b/bootstrap/mkpause index 874efcffc..6d969010b 100755 --- a/bootstrap/mkpause +++ b/bootstrap/mkpause @@ -1,4 +1,15 @@ #!/usr/bin/env perl + +# If you're building a test PAUSE to test this automation, this is the place to +# start. This program requires a DigitalOcean API token, which is used to +# create a VM. This program then copies `selfconfig-root` to that VM and runs +# it. `selfconfig-root` will install packages, create users, and configure +# services. One of the created users is `pause`, which will be used to run the +# `selfconfig-pause` program found in this directory. +# +# For higher-level and more detailed view, check out the README in this +# directory. + use v5.36.0; use lib 'lib'; diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index 9ffd5aefb..490e439d6 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -1,4 +1,17 @@ #!/usr/bin/perl + +# Normally, you won't run this program by hand. Instead, it's run indirectly +# by mkpause, which is run by a human. mkpause will create a new VM, copy +# selfconfig-root to it, and remotely run that program as root. +# selfconfig-root, in turn, will create a low-privilege user (pause) and run +# *this* program on the VM. +# +# This program's job is to install the Perl environment needed for PAUSE +# and then to configure PAUSE for operation on this VM. +# +# For higher-level and more detailed view, check out the README in this +# directory. + use v5.36.0; use Carp qw(croak); diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 1aff07092..86ad5f059 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -1,4 +1,18 @@ #!/usr/bin/perl + +# Normally, you won't run this program by hand. Instead, it's run by mkpause, +# which is run by a human. mkpause will create a new VM, copy this program to +# it, and then remotely execute this program as root. +# +# This program's job is to set up the remote machine to become a PAUSE. It +# installs the needed packages, creates needed unix accounts, sets up and +# starts services, and also runs the selfconfig-pause program. +# selfconfig-pause is found right next to this selfconfig-root in the PAUSE +# repository, and runs as the pause user. +# +# For higher-level and more detailed view, check out the README in this +# directory. + use v5.36.0; use warnings; From 8a6efb0d9b3955196036db90fdd096dc54525f8a Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 11:20:52 +0100 Subject: [PATCH 50/78] bootstrap: add a README.md --- bootstrap/README.md | 56 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 bootstrap/README.md diff --git a/bootstrap/README.md b/bootstrap/README.md new file mode 100644 index 000000000..e25f42507 --- /dev/null +++ b/bootstrap/README.md @@ -0,0 +1,56 @@ +# PAUSE Bootstrap + +This directory is a collection of programs to make it easy to stand up a +complete PAUSE instance, either for testing or for preparing a new PAUSE +instance to be used in production. + +For testing, the simplest thing might be for you to use `mkpause` which has +roughly this usage. + +``` +--username STR (or -u) your username; defaults to $ENV{USER} +--size STR slug for Digital Ocean droplet +--box-ident STR (or -i) identifying part of box name; defaults to --username + +--plenv-url STR (or -P) URL to a tar.bz2 file to use for plenv + +--certbot-staging (or -C) use the staging version of certbot +--enable-mail (or -m) set up postfix config for outbound mail + +--destroy destroy the box if it does exist +``` + +In general, you don't need to provide any options. If your local computer's +unix account is `hans` then you'll get a box named `hans.unpause.your-domain`, +where `your-domain` is the DigitalOcean domain you've set aside for this work. +That means you need a DigitalOcean account and an API key set in the `DO_TOKEN` +environment variable. + +`mkpause` will create a VM and then copy the `selfconfig-root` program to the +VM. It will run `selfconfig-root` as the root user. That program will install +a bunch of apt packages, configure the firewall, create non-root users, and +then run the program `selfconfig-pause` as the new `pause` user. When that's +done, cronjobs and systemd services will be installed and running, and you'll +be able to log into the web interface. + +**Admin user**: The bootstrap program will also create an admin account in +PAUSE for you, with the username and password both set to the value of +`--username`. This is useful for testing, but if you're preparing a new PAUSE +instance, make sure you delete it. + +**certbot**: By default, certbot will generate a real, trusted certificate for +the new VM. That's useful to test with your web browser without security +alerts. The down side is that you can only generate a limited number of Let's +Encrypt certificates before being rate limited. Pass `--certbot-staging` to +use the staging certbot server. This is a real certificate, but isn't trusted +by default in common certificate roots, so your browser will complain. On the +other hand, you can generate quite a lot of them without being rate-limited. + +**plenv**: The bootstrap process will default to building perl from source and +installing all of PAUSE's depenedencies using `cpanm`. On our usual VM size, +this takes around ten minutes. It's useful to save a copy of the +`~pause/.plenv` as an archive file so avoid waiting on build and install. Put +that archive, as a `tar.bz2` file, somewhere on the web, and then provide the +URL to it as the `--plenv-url` option. As of 2024-04-26, a workable archive +can be found at https://dot-plenv.nyc3.digitaloceanspaces.com/dot-plenv.tar.bz2 + From 1023f4c899001d2a09683753022995508d42edfc Mon Sep 17 00:00:00 2001 From: Matthew Horsfall Date: Fri, 26 Apr 2024 06:37:25 -0400 Subject: [PATCH 51/78] pause-web.service: We are pause_2017, not pause_1999! --- services/pause-web.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/pause-web.service b/services/pause-web.service index 62dd8af89..c94b629d8 100644 --- a/services/pause-web.service +++ b/services/pause-web.service @@ -16,7 +16,7 @@ Group=pause WorkingDirectory=/home/pause/pause # Should not daemonize -ExecStart=/home/pause/.plenv/shims/plackup --port 5000 +ExecStart=/home/pause/.plenv/shims/plackup --port 5000 app_2017.psgi Restart=on-failure RestartSec=30 From 6863b73a43bea2282ae295c2210195c405c4b5c2 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 11:45:13 +0100 Subject: [PATCH 52/78] bootstrap: fix HTTP_ERRORLOG config setting --- bootstrap/selfconfig-pause | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index 490e439d6..efe63a8ab 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -121,6 +121,7 @@ my $config_file_contents = <<~'END'; $Config->{FTPPUB} = '/home/pause/pub/PAUSE/'; $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; + $Config->{HTTP_ERRORLOG} = '/var/log/nginx/error.log'; $Config->{INCOMING_LOC} = '/home/pause/incoming'; $Config->{MLROOT} = '/home/pause/pub/PAUSE/authors/id/'; $Config->{ML_CHOWN_USER} = 'unsafe'; From 81d082d38ac383a2405f3fe5fbfcb7272ab0eff1 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 11:53:20 +0100 Subject: [PATCH 53/78] PAUSE: eliminate unused BATCH_SIG_HOME config setting --- lib/PAUSE.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/PAUSE.pm b/lib/PAUSE.pm index 0fb7c455b..abc31f836 100644 --- a/lib/PAUSE.pm +++ b/lib/PAUSE.pm @@ -123,7 +123,6 @@ $PAUSE::Config ||= CHECKSUMS_SIGNING_PROGRAM => ('gpg'), CHECKSUMS_SIGNING_ARGS => '--homedir /home/puppet/pause-private/gnupg-pause-batch-signing-home --clearsign --default-key ', CHECKSUMS_SIGNING_KEY => '450F89EC', - BATCH_SIG_HOME => "/home/puppet/pause-private/gnupg-pause-batch-signing-home", MIN_MTIME_CHECKSUMS => 1300000000, # invent a threshold for oldest mtime HAVE_PERLBAL => 1, ZCAT_PATH => (List::Util::first { -x $_ } ("/bin/zcat", "/usr/bin/zcat" )), From 47ec4bece431b9f62cd67d957b0701fbbe9fa4ca Mon Sep 17 00:00:00 2001 From: Matthew Horsfall Date: Fri, 26 Apr 2024 06:56:50 -0400 Subject: [PATCH 54/78] bootstrap: paused: Don't use ftp to fetch uploaded files --- bootstrap/selfconfig-pause | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index efe63a8ab..0e17340ca 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -122,6 +122,7 @@ my $config_file_contents = <<~'END'; $Config->{FTPPUB} = '/home/pause/pub/PAUSE/'; $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; $Config->{HTTP_ERRORLOG} = '/var/log/nginx/error.log'; + $Config->{INCOMING} = "file://home/pause/incoming/"; $Config->{INCOMING_LOC} = '/home/pause/incoming'; $Config->{MLROOT} = '/home/pause/pub/PAUSE/authors/id/'; $Config->{ML_CHOWN_USER} = 'unsafe'; From 42cdbbae40f6d53e4d9915d79a8c6d14ec7a4de1 Mon Sep 17 00:00:00 2001 From: Matthew Horsfall Date: Fri, 26 Apr 2024 07:31:53 -0400 Subject: [PATCH 55/78] mkpause: Check for ssh banner before continuing on Otherwise, sometimes the next thing we do trying to talk to ssh fails and the setup breaks. --- bootstrap/mkpause | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/bootstrap/mkpause b/bootstrap/mkpause index 6d969010b..500d782cf 100755 --- a/bootstrap/mkpause +++ b/bootstrap/mkpause @@ -274,23 +274,48 @@ async sub do_destroy { async sub wait_for_port ($ip_addr, $port) { my $max_tries = 20; TRY: for my $try (1..$max_tries) { - my $socket; - eval { + my $done = eval { + my $socket; + $socket = await $loop->connect(addr => { family => 'inet', socktype => 'stream', port => 22, ip => $ip_addr, }); + + if ($socket) { + $Logger->log("We connected... let's see if we get a banner..."); + + $socket->blocking(1); + + local $SIG{ALRM} = sub { die "timed out\n" }; + alarm 5; + + my $banner; + $socket->recv($banner, 1024); + + alarm 0; + + if ($banner && $banner =~ /^SSH/) { + $Logger->log("Yup!"); + + return 1; + } + + $Logger->log("Nope, not yet."); + + return 0; + } }; - if ($socket) { - # We didn't need the connection, just to know it worked! - return 1; - } + alarm 0; + + return 1 if $done; my $error = $@; - if ($error !~ /Connection refused/) { + + if ($error && $error !~ /(Connection refused|timed out)/) { $Logger->log([ "weird error connecting to %s:22: %s", $ip_addr, From b429d781720c5e2a923a2385dab7b30d56322077 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 12:58:21 +0100 Subject: [PATCH 56/78] bootstrap: correctly configure gpg for signing --- bootstrap/selfconfig-pause | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index 0e17340ca..cbf8c8aa3 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -91,9 +91,20 @@ run_cmd(qw(git config --global user.email pause@pause.perl.org)); run_cmd(qw(git config --global user.name), 'PAUSE Daemon'); run_cmd(qw(git init --initial-branch master )); -# This imports a test key, which nobody should trust, and which has key id -# 6BA1716EFB099DB2. -- rjbs, 2024-04-25 -run_cmd(qw( gpg --import --armor /home/pause/pause/bootstrap/test-key.txt )); +{ + # This imports a test key, which nobody should trust, and which has key id + # 6BA1716EFB099DB2. -- rjbs, 2024-04-25 + path("/home/pause/pause-private/gnupg-pause-batch-signing-home") + ->mkdir + ->chmod(0700); + + run_cmd(qw( + gpg + --homedir /home/pause/pause-private/gnupg-pause-batch-signing-home + --import + --armor /home/pause/pause/bootstrap/test-key.txt + )); +} my $config_file_contents = <<~'END'; use strict; From 52dfdfb2e418f5a705bbcd6471dc6b0ca84d3ce9 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 26 Apr 2024 02:12:23 -0700 Subject: [PATCH 57/78] bootstrap: configure firewall on PAUSE box Use ufw, not firewalld, because it's significantly easier to manage for simple things. --- bootstrap/selfconfig-root | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 86ad5f059..9f950a94a 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -76,10 +76,18 @@ my @required_debs = qw( unzip zlib1g-dev libsasl2-modules + ufw ); run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y), @required_debs); +# Some packages we just don't want. +my @unwanted_debs = qw( + firewalld +); + +run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 remove -y), @unwanted_debs); + require Getopt::Long::Descriptive; my ($opt, $usage) = Getopt::Long::Descriptive::describe_options( @@ -231,6 +239,13 @@ unlink('/etc/nginx/sites-enabled/default'); symlink("/etc/nginx/sites-available/$hostname", "/etc/nginx/sites-enabled/$hostname") or die "can't symlink nginx conf: $!"; +# Firewall config +run_cmd(qw(ufw allow http)); +run_cmd(qw(ufw allow https)); +run_cmd(qw(ufw allow rsync)); +run_cmd(qw(ufw allow ssh)); +run_cmd(qw(ufw --force enable)); + # Install ssl cert run_cmd( qw(sudo certbot --nginx -d), From a4a516d4bbd65ad03a608d188cfe801f71699f22 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 26 Apr 2024 03:37:03 -0700 Subject: [PATCH 58/78] bootstrap: add partitioning and mount config options --- bootstrap/selfconfig-root | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 9f950a94a..47fd51f99 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -63,14 +63,12 @@ run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 update)); my @required_debs = qw( build-essential certbot - default-libmysqlclient-dev git libdb-dev libexpat1-dev libgetopt-long-descriptive-perl libpath-tiny-perl libssl-dev - mariadb-server nginx python3-certbot-nginx unzip @@ -105,6 +103,7 @@ my ($opt, $usage) = Getopt::Long::Descriptive::describe_options( [ 'relay-port=s', "relay port for smtp" ], [ 'relay-username=s', "relay sasl username for smtp" ], [ 'relay-password=s', "relay sasl password for smtp" ], + [ 'volume-group=s', "volume group for data" ], ); my @mopts = qw(relay_host relay_port relay_username relay_password); @@ -133,6 +132,36 @@ if ($opt->plenv_url) { run_cmd('curl', $opt->plenv_url, '--output', '/tmp/plenv-tarball.tar.bz2'); } +require Path::Tiny; + +# Partitioning! + +run_cmd(qw(mkdir -p /data/mysql /data/pause)); +if (-e "/usr/sbin/lvcreate" && $opt->volume_group) { + my $vg = $opt->volume_group; + run_cmd(qw(lvcreate -L4G), qq($vg), qw(-n mysql)); + run_cmd(qw(lvcreate -L50G), qq($vg), qw(-n pause)); + Path::Tiny::path("/etc/fstab")->append(<<~EOF); +/dev/$vg/mysql /data/mysql ext4 defaults 0 2 +/dev/$vg/pause /data/pause ext4 defaults 0 2 +EOF + run_cmd(qw(systemctl daemon-reload)); + run_cmd(qw(mkfs.ext4 -j), qq(/dev/$vg/mysql)); + run_cmd(qw(mkfs.ext4 -j), qq(/dev/$vg/pause)); + run_cmd(qw(mount /data/mysql)); + run_cmd(qw(mount /data/pause)); +} +run_cmd(qw(mkdir /data/pause/ftp)); +run_cmd(qw(ln -s /data/mysql /var/lib/mysql)); +run_cmd(qw(ln -s /data/pause/ftp /home/ftp)); + +# Mariadb has to be installed _after_ partitioning. +run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y), + qw( + mariadb-server + default-libmysqlclient-dev + )); + run_cmd(qw( sudo -u pause git clone -b unpause https://git@github.com/rjbs/pause/ /home/pause/pause )); @@ -232,7 +261,6 @@ server { } END -require Path::Tiny; Path::Tiny::path("/etc/nginx/sites-available/$hostname")->spew($nginx_config); unlink('/etc/nginx/sites-enabled/default'); From ad2fb68acb4a93de19ec3cd26425841dde7aab51 Mon Sep 17 00:00:00 2001 From: Matthew Horsfall Date: Fri, 26 Apr 2024 08:52:15 -0400 Subject: [PATCH 59/78] bootstrap: Set mailname properly (no quotes!) Otherwise this breaks sending of mail that self-determines our hostname. Also, fix main_mailer_type to be correct (which means we also need relayhost). This is kinda silly, because we throw out the generated postfix conf and rewrite it afterwards... but leaves us in a place where we could use the debian provided config in the future (if we want). --- bootstrap/selfconfig-root | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 47fd51f99..63b08eeb4 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -328,16 +328,22 @@ run_cmd( Path::Tiny::path("/home/pause/pause/cron/CRONTAB.ROOT")->copy("/etc/cron.d/pause"); if ($opt->enable_mail) { + my $relayhost = $opt->relay_host; + my $relayport = $opt->relay_port; + system( - q{echo "postfix postfix/main_mailer_type string 'Internet with smarthost'" | debconf-set-selections} + q{echo postfix postfix/main_mailer_type select "Internet with smarthost" | debconf-set-selections} ); - croak "failed to update debconf for main_mailer_type" if $?; system( - qq{echo "postfix postfix/mailname string '$hostname'" | debconf-set-selections} + qq{echo postfix postfix/mailname string $hostname | debconf-set-selections} ); + croak "failed to update debconf for mailname" if $?; + system( + qq{echo postfix postfix/relayhost string \\[$relayhost\\]:$relayport | debconf-set-selections} + ); croak "failed to update debconf for mailname" if $?; run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y postfix)); @@ -350,9 +356,6 @@ if ($opt->enable_mail) { $maincf =~ s{inet_interfaces = all}{inet_interfaces = localhost} or warn "!!! Failed to replace inet_interfaces !!!\n\n"; - my $relayhost = $opt->relay_host; - my $relayport = $opt->relay_port; - $maincf .= <<~EOF; relayhost = [$relayhost]:$relayport From b6bdb00c3d6fef40f5878866ed61d0e301492efd Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 14:17:14 +0100 Subject: [PATCH 60/78] bootstrap: load some options from config --- bootstrap/.gitignore | 1 + bootstrap/mkpause | 31 +++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 bootstrap/.gitignore diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore new file mode 100644 index 000000000..18218c8c2 --- /dev/null +++ b/bootstrap/.gitignore @@ -0,0 +1 @@ +.mkpause diff --git a/bootstrap/mkpause b/bootstrap/mkpause index 500d782cf..549e18170 100755 --- a/bootstrap/mkpause +++ b/bootstrap/mkpause @@ -19,19 +19,33 @@ use Future::AsyncAwait; use Getopt::Long::Descriptive; use IO::Async::Loop; use Log::Dispatchouli; +use YAML::XS qw(LoadFile); + +my %default; +my $file = $ENV{PAUSE_MKPAUSE_CONFIG} // ".mkpause"; +if (-e $file) { + my ($config) = LoadFile($file); + %default = %$config; +} my ($opt, $usage) = describe_options( '%c %o', - [ 'username|u=s', "your username; defaults to $ENV{USER}", - { default => $ENV{USER} // die 'no USER env var!' } ], + [ 'username|u=s', "your username; defaults to $ENV{USER}", + { default => $default{username} // $ENV{USER} // die 'no USER env var!' } ], # For serious business, we like c-16. -- rjbs, 2024-04-19 - [ 'size=s', "slug for Digital Ocean droplet", { default => 'g-4vcpu-16gb' } ], + [ 'size=s', "slug for DigitalOcean droplet", + { default => $default{size} // 'g-4vcpu-16gb' } ], + [ 'project_id=s', "if given, a DigitalOcean project id to add the VM to", + { default => $default{'project-id'} } ], + [ 'box-ident|i=s', "identifying part of box name; defaults to --username" ], [], - [ 'plenv-url|P=s', "URL to a tar.bz2 file to use for plenv (it's a shortcut)" ], + [ 'plenv-url|P=s', "URL to a tar.bz2 file to use for plenv (it's a shortcut)", + { default => $default{'plenv-url'} } ], [], - [ 'certbot-staging|C', 'use the staging version of certbot' ], + [ 'certbot-staging|C!', 'use the staging version of certbot', + { default => $default{'certbot-staging'} } ], [ 'enable-mail|m', "set up postfix config for outbound mail" ], [ 'relay-host=s', "relay host for smtp" ], [ 'relay-port=s', "relay port for smtp" ], @@ -72,7 +86,7 @@ my $Logger = Log::Dispatchouli->new({ my $loop = IO::Async::Loop->new; -my $TOKEN = $ENV{DO_TOKEN} // die "no token, no milk\n"; +my $TOKEN = $ENV{DO_TOKEN} // $default{'api-token'} // die "no DigitalOcean API token\n"; my $dobby = Dobby::Client->new( bearer_token => $TOKEN, @@ -83,7 +97,6 @@ $loop->add($dobby); my $domain = "fastmail.dev"; my $username = $opt->username; my $boxname = ($opt->box_ident // $username) . ".unpause"; -my $project_id = q{62113225-81be-4538-8408-f42f54f6c93f}; # PAUSE 2024 my $todo = __PACKAGE__->can("do_" . $opt->mode); @@ -175,7 +188,9 @@ async sub do_create { $Logger->log([ "Droplet is now up on %s...", $ip_addr ]); - await $dobby->add_droplet_to_project($droplet->{id}, $project_id); + if ($opt->project_id) { + await $dobby->add_droplet_to_project($droplet->{id}, $opt->project_id); + } $Logger->log("updating DNS names for $boxname"); From 6a34e9ff9c78cb2cae40d8828b85ea36bd10df90 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 14:40:04 +0100 Subject: [PATCH 61/78] bootstrap/import-pause-data: fix db name we import to --- bootstrap/import-pause-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/import-pause-data b/bootstrap/import-pause-data index 598c347cd..9de999510 100755 --- a/bootstrap/import-pause-data +++ b/bootstrap/import-pause-data @@ -40,7 +40,7 @@ bunzip2 authen_pausedump.current.bz2 pv authen_pause.current | \ perl -ple 's/^CHANGE MASTER.*//' | \ - sudo mysql authen + sudo mysql authen_pause echo 'SET GLOBAL innodb_flush_log_at_trx_commit=1' | mysql From 765de69099493a1324be9e4e427623108c331736 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 14:51:53 +0100 Subject: [PATCH 62/78] bootstrap: capture mail to a maildir in ~pause --- bootstrap/selfconfig-pause | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index cbf8c8aa3..ee5bc01d6 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -78,6 +78,7 @@ for my $path (qw( /home/pause/pub/PAUSE/modules /home/pause/pub/PAUSE/PAUSE-git /home/pause/run + /home/pause/testmail /home/pause/tmp /tmp/pause_1999 )) { @@ -110,7 +111,8 @@ my $config_file_contents = <<~'END'; use strict; package PAUSE; - $ENV{EMAIL_SENDER_TRANSPORT} = 'DevNull'; + $ENV{EMAIL_SENDER_TRANSPORT} = 'Maildir'; + $ENV{EMAIL_SENDER_TRANSPORT_dir} = '/home/pause/testmail'; our $Config; $Config->{AUTHEN_BACKUP_DIR} = "/home/pause/db-backup"; From 818bda69b6d07618692a3ca287c267f218034888 Mon Sep 17 00:00:00 2001 From: Matthew Horsfall Date: Fri, 26 Apr 2024 09:13:12 -0400 Subject: [PATCH 63/78] mkpause: add --list|l to list boxes And condense some reused code. --- bootstrap/mkpause | 62 ++++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/bootstrap/mkpause b/bootstrap/mkpause index 549e18170..3899f3817 100755 --- a/bootstrap/mkpause +++ b/bootstrap/mkpause @@ -30,6 +30,8 @@ if (-e $file) { my ($opt, $usage) = describe_options( '%c %o', + [ 'list|l', "list boxes that are running then quit" ], + [ 'username|u=s', "your username; defaults to $ENV{USER}", { default => $default{username} // $ENV{USER} // die 'no USER env var!' } ], @@ -63,18 +65,11 @@ my ($opt, $usage) = describe_options( ], ); -my @mopts = qw(relay_host relay_port relay_username relay_password); - -if ($opt->enable_mail) { - my @m; - - for my $setting (@mopts) { - push @m, $setting unless $opt->$setting; - } - - $_ =~ s/_/-/g for @m; +my sub ip_addr_for ($droplet) { + my ($ip_addr) = map { $_->{ip_address} } grep { $_->{type} eq 'public'} + $droplet->{networks}{v4}->@*; - die "--enable-mail requires @m\n" if @m; + return $ip_addr; } my $Logger = Log::Dispatchouli->new({ @@ -94,6 +89,38 @@ my $dobby = Dobby::Client->new( $loop->add($dobby); +if ($opt->list) { + my @droplets = $dobby->get_droplets_with_tag('unpause')->get; + + if (@droplets) { + $Logger->log("Droplets:"); + + my %drops = map {; $_->{name} => ip_addr_for($_) } @droplets; + + for my $k (sort keys %drops) { + $Logger->log([" %-20s %s", $k, $drops{$k}]); + } + } else { + $Logger->log("No unpause droplets found"); + } + + exit; +} + +my @mopts = qw(relay_host relay_port relay_username relay_password); + +if ($opt->enable_mail) { + my @m; + + for my $setting (@mopts) { + push @m, $setting unless $opt->$setting; + } + + $_ =~ s/_/-/g for @m; + + die "--enable-mail requires @m\n" if @m; +} + my $domain = "fastmail.dev"; my $username = $opt->username; my $boxname = ($opt->box_ident // $username) . ".unpause"; @@ -107,24 +134,15 @@ await $todo->(); #---( cut here )--- -my sub ip_addr_for ($droplet) { - my ($ip_addr) = map { $_->{ip_address} } grep { $_->{type} eq 'public'} - $droplet->{networks}{v4}->@*; - - return $ip_addr; -} - async sub do_create { { my @droplets = await $dobby->get_droplets_with_tag('unpause'); my ($droplet) = grep {; $_->{name} eq $boxname } @droplets; if ($droplet) { - my ($net) = grep {; - $_->{type} eq 'public' - } $droplet->{networks}{v4}->@*; + my $ip = ip_addr_for($droplet); - my $extra = $net ? " at root\@$net->{ip_address}" : ""; + my $extra = $ip ? " at root\@$ip" : ""; die "box already exists$extra\n"; } From 22f39143e385bc33108d51120f57b58b8f308008 Mon Sep 17 00:00:00 2001 From: Matthew Horsfall Date: Fri, 26 Apr 2024 11:14:44 -0400 Subject: [PATCH 64/78] pause-web: Use starman instead of a single process ... So we can handle user load, etc --- cpanfile | 1 + services/pause-web.service | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cpanfile b/cpanfile index ed725cbd9..b0009040a 100644 --- a/cpanfile +++ b/cpanfile @@ -47,6 +47,7 @@ requires 'Plack::Middleware::ReverseProxy'; requires 'Plack::Middleware::ServerStatus::Tiny'; requires 'Process::Status'; requires 'Set::Crontab'; +requires 'Starman'; requires 'String::Random'; requires 'Text::Format'; requires 'Text::Markdown::Hoedown'; diff --git a/services/pause-web.service b/services/pause-web.service index c94b629d8..92d842652 100644 --- a/services/pause-web.service +++ b/services/pause-web.service @@ -16,7 +16,7 @@ Group=pause WorkingDirectory=/home/pause/pause # Should not daemonize -ExecStart=/home/pause/.plenv/shims/plackup --port 5000 app_2017.psgi +ExecStart=/home/pause/.plenv/shims/plackup -s Starman --port 5000 app_2017.psgi Restart=on-failure RestartSec=30 From c34058b47d04600e2ffa7c840e3058ed8f8e95ef Mon Sep 17 00:00:00 2001 From: Matthew Horsfall Date: Fri, 26 Apr 2024 11:18:13 -0400 Subject: [PATCH 65/78] bootstrap: Remove mojolicious pin, pin DBD::mysql lower Pause should work with newest mojolicious now. DBD::mysql needs to be dropped a few versions down to match pause2, otherwise cached dbh handles aren't usable after an explicit disconnect, and you see errors in the logs like: [2024-04-26 07:04:29] [warn] DBD::mysql::db prepare failed: Statement not active at /home/pause/pause/lib/pause_2017/PAUSE/Web/Plugin/ConfigPerRequest.pm line 96. [2024-04-26 07:04:29] [error] DBD::mysql::db prepare failed: Statement not active at /home/pause/pause/lib/pause_2017/PAUSE/Web/Plugin/ConfigPerRequest.pm line 96. [2024-04-26 07:04:29] [error] DBD::mysql::db prepare failed: Statement not active at /home/pause/pause/lib/pause_2017/PAUSE/Web/Plugin/ConfigPerRequest.pm line 96. --- bootstrap/selfconfig-pause | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index ee5bc01d6..23f443c7e 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -59,9 +59,11 @@ if (-e '/tmp/plenv-tarball.tar.bz2') { # install cpanm for perl dep management run_cmd(qw( /home/pause/.plenv/bin/plenv install-cpanm )); - # We need to pin these for now - run_cmd(qw( /home/pause/.plenv/shims/cpanm -n Mojolicious@8.72 )); - run_cmd(qw( /home/pause/.plenv/shims/cpanm -n DBD::mysql@4.052 )); + # Pin DBD::mysql to 4.050. Newer doesn't work for some reason + # last I knew (mariadb problems?). And 4.052 breaks reconnecting + # after calls to disconnect, so we can't use that. 4.050 is what + # is in pause2 right now, so this should be fine. + run_cmd(qw( /home/pause/.plenv/shims/cpanm -n DBD::mysql@4.050 )); run_cmd(qw( /home/pause/.plenv/shims/cpanm -n --installdeps . )); } From 99b674297d5cc447e1170537088bd403c04f0fea Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Fri, 26 Apr 2024 16:53:56 +0100 Subject: [PATCH 66/78] bootstrap: move --list into the mode switches --- bootstrap/mkpause | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bootstrap/mkpause b/bootstrap/mkpause index 3899f3817..25e30fc3f 100755 --- a/bootstrap/mkpause +++ b/bootstrap/mkpause @@ -30,8 +30,6 @@ if (-e $file) { my ($opt, $usage) = describe_options( '%c %o', - [ 'list|l', "list boxes that are running then quit" ], - [ 'username|u=s', "your username; defaults to $ENV{USER}", { default => $default{username} // $ENV{USER} // die 'no USER env var!' } ], @@ -59,6 +57,7 @@ my ($opt, $usage) = describe_options( default => 'create', one_of => [ [ 'create', 'create the box if it does not exist' ], + [ 'list|l', 'list boxes that are running then quit' ], [ 'destroy', 'destroy the box if it does exist' ], ], } From 10f81ea15e0a270cd3b5d9ecd74c9f2814f74823 Mon Sep 17 00:00:00 2001 From: Robert <43223+rspier@users.noreply.github.com> Date: Fri, 26 Apr 2024 13:06:22 +0100 Subject: [PATCH 67/78] bootstrap: fix import of authen_pause data --- bootstrap/import-pause-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/import-pause-data b/bootstrap/import-pause-data index 9de999510..ad5a80aa4 100755 --- a/bootstrap/import-pause-data +++ b/bootstrap/import-pause-data @@ -38,7 +38,7 @@ pv moddump.current | \ bunzip2 authen_pausedump.current.bz2 -pv authen_pause.current | \ +pv authen_pausedump.current | \ perl -ple 's/^CHANGE MASTER.*//' | \ sudo mysql authen_pause From 2a3bc2bb3b712f8e09c960e0dc33c0c75cec1727 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 27 Apr 2024 09:48:35 +0100 Subject: [PATCH 68/78] bootstrap: send cron output to ~pause/log just the one cron job --- cron/CRONTAB.ROOT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index 48c107ba9..34b939276 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -32,7 +32,7 @@ PAUSE_ROOT=/home/pause/pub/PAUSE 21 */6 * * * pause $PAUSE_REPO/cron/rm_stale_links 23 07,13,19,01 * * * pause $PAUSE_REPO/cron/run_mirrors.sh 22 * * * * pause $PAUSE_REPO/cron/sync-04pause.pl -10 09,15,21,03 * * * pause cd $PAUSE_ROOT/PAUSE-git && (git gc && git push -u origin master) >> /var/log/git-gc-push.out +10 09,15,21,03 * * * pause cd $PAUSE_ROOT/PAUSE-git && (git gc && git push -u origin master) >> /home/pause/log/git-gc-push.out 18 * * * * pause $PAUSE_REPO/cron/cron-p6daily.pl 46 0,6,12,18 * * * pause perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl > $PAUSE_REPO/bin/indexscripts.pl.out 2>&1 7 2 * * 0 pause perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl -f From e0c369aaab73670a9595cae96e5d4065152af718 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 26 Apr 2024 10:25:34 -0700 Subject: [PATCH 69/78] bootstrap: update paths to get rid of /home/ftp (also, fixes for other home-vs-data location) --- bootstrap/selfconfig-pause | 10 +++++----- bootstrap/selfconfig-root | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index 23f443c7e..6bd5bb51d 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -134,21 +134,21 @@ my $config_file_contents = <<~'END'; $Config->{CRONPATH} = '/home/pause/pause/cron/'; - $Config->{FTPPUB} = '/home/pause/pub/PAUSE/'; - $Config->{GITROOT} = '/home/pause/pub/PAUSE/PAUSE-git'; + $Config->{FTPPUB} = '/data/pause/pub/PAUSE/'; + $Config->{GITROOT} = '/data/pause/pub/PAUSE/PAUSE-git'; $Config->{HTTP_ERRORLOG} = '/var/log/nginx/error.log'; $Config->{INCOMING} = "file://home/pause/incoming/"; $Config->{INCOMING_LOC} = '/home/pause/incoming'; - $Config->{MLROOT} = '/home/pause/pub/PAUSE/authors/id/'; + $Config->{MLROOT} = '/data/pause/pub/PAUSE/authors/id/'; $Config->{ML_CHOWN_USER} = 'unsafe'; $Config->{ML_CHOWN_GROUP} = 'unsafe'; $Config->{ML_MIN_FILES} = 1; $Config->{ML_MIN_INDEX_LINES} = 0; $Config->{PAUSE_LOG} = "/home/pause/log/paused.log"; $Config->{PAUSE_LOG_DIR} = "/home/pause/log/"; - $Config->{PAUSE_PUBLIC_DATA} = '/home/pause/pub/PAUSE/PAUSE-data'; + $Config->{PAUSE_PUBLIC_DATA} = '/data/pause/pub/PAUSE/PAUSE-data'; $Config->{PID_DIR} = "/home/pause/pid/"; - $Config->{TMP} = "/home/pause/tmp/"; + $Config->{TMP} = "/data/pause/tmp/"; END $config_file_contents =~ s/%%AUTHUSER_PW%%/$opt->authuser_pw/e; diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 63b08eeb4..203439fe5 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -136,7 +136,9 @@ require Path::Tiny; # Partitioning! -run_cmd(qw(mkdir -p /data/mysql /data/pause)); +run_cmd(qw(mkdir -p /data/mysql /data/pause/pub)); +run_cmd(qw(mkdir -p /data/mysql /data/pause/incoming)); +run_cmd(qw(mkdir -p /data/mysql /data/pause/tmp)); if (-e "/usr/sbin/lvcreate" && $opt->volume_group) { my $vg = $opt->volume_group; run_cmd(qw(lvcreate -L4G), qq($vg), qw(-n mysql)); @@ -151,9 +153,8 @@ EOF run_cmd(qw(mount /data/mysql)); run_cmd(qw(mount /data/pause)); } -run_cmd(qw(mkdir /data/pause/ftp)); run_cmd(qw(ln -s /data/mysql /var/lib/mysql)); -run_cmd(qw(ln -s /data/pause/ftp /home/ftp)); +run_cmd(qw(ln -s /data/pause /home/ftp)); # Mariadb has to be installed _after_ partitioning. run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y), From 44462792140d3e3a6bafc222e5294e3dd9c7f508 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 26 Apr 2024 10:10:45 -0700 Subject: [PATCH 70/78] put mysql data in /data/mysql/mysql otherwise there's a lost+found file in /data/mysql (root of the mount) and mysqld complains. --- bootstrap/selfconfig-root | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 203439fe5..8c075be4c 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -135,10 +135,15 @@ if ($opt->plenv_url) { require Path::Tiny; # Partitioning! +for my $dir (qw( + /data/mysql/mysql + /data/pause/pub + /data/pause/incoming + /data/pause/tmp +)) { + Path::Tiny::path($dir)->mkdir; +} -run_cmd(qw(mkdir -p /data/mysql /data/pause/pub)); -run_cmd(qw(mkdir -p /data/mysql /data/pause/incoming)); -run_cmd(qw(mkdir -p /data/mysql /data/pause/tmp)); if (-e "/usr/sbin/lvcreate" && $opt->volume_group) { my $vg = $opt->volume_group; run_cmd(qw(lvcreate -L4G), qq($vg), qw(-n mysql)); @@ -153,8 +158,10 @@ EOF run_cmd(qw(mount /data/mysql)); run_cmd(qw(mount /data/pause)); } -run_cmd(qw(ln -s /data/mysql /var/lib/mysql)); -run_cmd(qw(ln -s /data/pause /home/ftp)); + +run_cmd(qw(mkdir /data/pause/ftp)); +run_cmd(qw(ln -s /data/mysql/mysql /var/lib/mysql)); +run_cmd(qw(ln -s /data/pause/ftp /home/ftp)); # Mariadb has to be installed _after_ partitioning. run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y), From 4f39b016e0486f35ac425460243a624cd1e81f24 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 27 Apr 2024 10:12:23 +0100 Subject: [PATCH 71/78] bootstrap: fix permissions on /data/pause --- bootstrap/selfconfig-root | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 8c075be4c..a7a94e0cb 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -135,13 +135,16 @@ if ($opt->plenv_url) { require Path::Tiny; # Partitioning! +Path::Tiny::path("/data/mysql/mysql")->mkdir; for my $dir (qw( - /data/mysql/mysql + /data/pause + /data/pause/ftp /data/pause/pub /data/pause/incoming /data/pause/tmp )) { Path::Tiny::path($dir)->mkdir; + run_cmd('chown', 'pause:', $dir); } if (-e "/usr/sbin/lvcreate" && $opt->volume_group) { @@ -159,7 +162,6 @@ EOF run_cmd(qw(mount /data/pause)); } -run_cmd(qw(mkdir /data/pause/ftp)); run_cmd(qw(ln -s /data/mysql/mysql /var/lib/mysql)); run_cmd(qw(ln -s /data/pause/ftp /home/ftp)); From 1ff2a2ce522f29ba14dc685ee9a3542adc15e773 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 27 Apr 2024 10:18:42 +0100 Subject: [PATCH 72/78] bootstrap: try to fix permissions on /data --- bootstrap/selfconfig-pause | 25 +++++++++++++++++-------- bootstrap/selfconfig-root | 23 ++++++++++------------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index 6bd5bb51d..6ac6474d7 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -70,24 +70,33 @@ if (-e '/tmp/plenv-tarball.tar.bz2') { chdir("/home/pause/pause") or die "can't chdir to ~pause/pause: $!"; +for my $dir (qw( + /data/pause + /data/pause/ftp + /data/pause/pub + /data/pause/incoming + /data/pause/tmp +)) { + Path::Tiny::path($dir)->mkdir; +} + # Set up pause config for my $path (qw( - /home/pause/incoming + /data/pause/pub/PAUSE/authors/id + /data/pause/pub/PAUSE/modules + /data/pause/pub/PAUSE/PAUSE-git + /home/pause/log /home/pause/pause-private/lib /home/pause/pid - /home/pause/pub/PAUSE/authors/id - /home/pause/pub/PAUSE/modules - /home/pause/pub/PAUSE/PAUSE-git /home/pause/run /home/pause/testmail - /home/pause/tmp /tmp/pause_1999 )) { path($path)->mkdir; } -chdir("/home/pause/pub/PAUSE/PAUSE-git") +chdir("/data/pause/pub/PAUSE/PAUSE-git") || die "couldn't chdir to PAUSE-git: $!"; run_cmd(qw(git config --global user.email pause@pause.perl.org)); @@ -137,8 +146,8 @@ my $config_file_contents = <<~'END'; $Config->{FTPPUB} = '/data/pause/pub/PAUSE/'; $Config->{GITROOT} = '/data/pause/pub/PAUSE/PAUSE-git'; $Config->{HTTP_ERRORLOG} = '/var/log/nginx/error.log'; - $Config->{INCOMING} = "file://home/pause/incoming/"; - $Config->{INCOMING_LOC} = '/home/pause/incoming'; + $Config->{INCOMING} = "file://data/pause/incoming/"; + $Config->{INCOMING_LOC} = '/data/pause/incoming'; $Config->{MLROOT} = '/data/pause/pub/PAUSE/authors/id/'; $Config->{ML_CHOWN_USER} = 'unsafe'; $Config->{ML_CHOWN_GROUP} = 'unsafe'; diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index a7a94e0cb..00e78c8f4 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -134,18 +134,10 @@ if ($opt->plenv_url) { require Path::Tiny; -# Partitioning! -Path::Tiny::path("/data/mysql/mysql")->mkdir; -for my $dir (qw( - /data/pause - /data/pause/ftp - /data/pause/pub - /data/pause/incoming - /data/pause/tmp -)) { - Path::Tiny::path($dir)->mkdir; - run_cmd('chown', 'pause:', $dir); -} +Path::Tiny::path("/data/mysql")->mkdir; + +Path::Tiny::path("/data/pause")->mkdir; +run_cmd("chown", "pause:", "/data/pause"); if (-e "/usr/sbin/lvcreate" && $opt->volume_group) { my $vg = $opt->volume_group; @@ -162,8 +154,10 @@ EOF run_cmd(qw(mount /data/pause)); } +# Partitioning! +Path::Tiny::path("/data/mysql/mysql")->mkdir; + run_cmd(qw(ln -s /data/mysql/mysql /var/lib/mysql)); -run_cmd(qw(ln -s /data/pause/ftp /home/ftp)); # Mariadb has to be installed _after_ partitioning. run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y), @@ -335,6 +329,9 @@ run_cmd( "--moduser-pw", $db_password_for{moduser}, ); +# XXX: I would like to not have or need this! -- rjbs, 2024-04-27 +run_cmd(qw(ln -s /data/pause/ftp /home/ftp)); + Path::Tiny::path("/home/pause/pause/cron/CRONTAB.ROOT")->copy("/etc/cron.d/pause"); if ($opt->enable_mail) { From 2fc8b15efddc15b09966654d1d15d8d05c6f30d0 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 27 Apr 2024 11:00:15 +0100 Subject: [PATCH 73/78] bootstrap: also chown after mounting with vg --- bootstrap/selfconfig-root | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 00e78c8f4..f597a697d 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -152,6 +152,10 @@ EOF run_cmd(qw(mkfs.ext4 -j), qq(/dev/$vg/pause)); run_cmd(qw(mount /data/mysql)); run_cmd(qw(mount /data/pause)); + + # We have to chown *again* after mounting. We don't *only* chown here + # because we only enter this branch when --volume-group was passed! + run_cmd("chown", "pause:", "/data/pause"); } # Partitioning! From 48f6483095dd2c229ff9566cd988b02df9b171f9 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 27 Apr 2024 11:13:13 +0100 Subject: [PATCH 74/78] bootstrap: fix path used in import program --- bootstrap/import-pause-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/import-pause-data b/bootstrap/import-pause-data index ad5a80aa4..a622579f9 100755 --- a/bootstrap/import-pause-data +++ b/bootstrap/import-pause-data @@ -26,7 +26,7 @@ touch /etc/PAUSE.CLOSED systemctl stop paused -sudo -u pause rsync --progress -av pause.perl.org::PAUSE/ /home/pause/pub/PAUSE/ +sudo -u pause rsync --progress -av pause.perl.org::PAUSE/ /data/pause/pub/PAUSE/ echo 'SET GLOBAL innodb_flush_log_at_trx_commit=2' | mysql From 725b31eed2b5f912f7d71d6bc7b142c439b811f7 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 27 Apr 2024 11:21:48 +0100 Subject: [PATCH 75/78] bootstrap: fix paths in rsync config --- bootstrap/selfconfig-root | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index f597a697d..5f143eeb7 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -305,25 +305,25 @@ Path::Tiny::path("/etc/rsyncd.conf")->spew(<<~'END'); timeout = 600 [PAUSE] - path = /home/pause/pub/PAUSE + path = /data/pause/pub/PAUSE [authors] - path = /home/pause/pub/PAUSE/authors + path = /data/pause/pub/PAUSE/authors [modules] - path = /home/pause/pub/PAUSE/modules + path = /data/pause/pub/PAUSE/modules [scripts] - path = /home/pause/pub/PAUSE/scripts/new + path = /data/pause/pub/PAUSE/scripts/new [pausedata] - path = /home/pause/pub/PAUSE/PAUSE-data + path = /data/pause/pub/PAUSE/PAUSE-data [pausecode] - path = /home/pause/pub/PAUSE/PAUSE-code + path = /data/pause/pub/PAUSE/PAUSE-code [pausegit] - path = /home/pause/pub/PAUSE/PAUSE-git + path = /data/pause/pub/PAUSE/PAUSE-git END run_cmd( From b68c4c417e673130c00ec0177fd05ee2cac58fed Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 27 Apr 2024 11:25:16 +0100 Subject: [PATCH 76/78] cron: fix paths in crontab --- cron/CRONTAB.ROOT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index 34b939276..4b4d68748 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -2,7 +2,7 @@ PATH=/home/pause/.plenv/shims:/usr/bin:/home/pause/pause/cron PAUSE_REPO=/home/pause/pause -PAUSE_ROOT=/home/pause/pub/PAUSE +PAUSE_ROOT=/data/pause/pub/PAUSE ## STUFF RJBS DID TO PUT THIS INTO UNPAUSE: ## * replace a bunch of paths: From 9cb0529895fb0cad23902c8156ed4ed454f3a795 Mon Sep 17 00:00:00 2001 From: Ricardo Signes Date: Sat, 27 Apr 2024 11:26:48 +0100 Subject: [PATCH 77/78] bootstrap: eliminate bogus mkdir /data/pause will exist, and pause user could not create it --- bootstrap/selfconfig-pause | 1 - 1 file changed, 1 deletion(-) diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause index 6ac6474d7..d352379a6 100755 --- a/bootstrap/selfconfig-pause +++ b/bootstrap/selfconfig-pause @@ -71,7 +71,6 @@ if (-e '/tmp/plenv-tarball.tar.bz2') { chdir("/home/pause/pause") or die "can't chdir to ~pause/pause: $!"; for my $dir (qw( - /data/pause /data/pause/ftp /data/pause/pub /data/pause/incoming From 24130dc7990667fb9dd8a3e7ca5cbe28cf1a0d6c Mon Sep 17 00:00:00 2001 From: Robert <43223+rspier@users.noreply.github.com> Date: Sat, 27 Apr 2024 12:06:02 +0100 Subject: [PATCH 78/78] /data/pause is the new /home/pause fix a confusion with paths --- bootstrap/selfconfig-root | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root index 5f143eeb7..793404f07 100755 --- a/bootstrap/selfconfig-root +++ b/bootstrap/selfconfig-root @@ -334,7 +334,7 @@ run_cmd( ); # XXX: I would like to not have or need this! -- rjbs, 2024-04-27 -run_cmd(qw(ln -s /data/pause/ftp /home/ftp)); +run_cmd(qw(ln -s /data/pause /home/ftp)); Path::Tiny::path("/home/pause/pause/cron/CRONTAB.ROOT")->copy("/etc/cron.d/pause");