diff --git a/bin/indexscripts.pl b/bin/indexscripts.pl index ef70cd2f2..3cc7a48cf 100755 --- a/bin/indexscripts.pl +++ b/bin/indexscripts.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/home/pause/.plenv/shims/perl # Build the scripts index for PAUSE # Original author: KSTAR # Last modified: $Date: 2003/12/13 05:52:51 $ diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore new file mode 100644 index 000000000..18218c8c2 --- /dev/null +++ b/bootstrap/.gitignore @@ -0,0 +1 @@ +.mkpause diff --git a/bootstrap/README.md b/bootstrap/README.md new file mode 100644 index 000000000..e25f42507 --- /dev/null +++ b/bootstrap/README.md @@ -0,0 +1,56 @@ +# PAUSE Bootstrap + +This directory is a collection of programs to make it easy to stand up a +complete PAUSE instance, either for testing or for preparing a new PAUSE +instance to be used in production. + +For testing, the simplest thing might be for you to use `mkpause` which has +roughly this usage. + +``` +--username STR (or -u) your username; defaults to $ENV{USER} +--size STR slug for Digital Ocean droplet +--box-ident STR (or -i) identifying part of box name; defaults to --username + +--plenv-url STR (or -P) URL to a tar.bz2 file to use for plenv + +--certbot-staging (or -C) use the staging version of certbot +--enable-mail (or -m) set up postfix config for outbound mail + +--destroy destroy the box if it does exist +``` + +In general, you don't need to provide any options. If your local computer's +unix account is `hans` then you'll get a box named `hans.unpause.your-domain`, +where `your-domain` is the DigitalOcean domain you've set aside for this work. +That means you need a DigitalOcean account and an API key set in the `DO_TOKEN` +environment variable. + +`mkpause` will create a VM and then copy the `selfconfig-root` program to the +VM. It will run `selfconfig-root` as the root user. That program will install +a bunch of apt packages, configure the firewall, create non-root users, and +then run the program `selfconfig-pause` as the new `pause` user. When that's +done, cronjobs and systemd services will be installed and running, and you'll +be able to log into the web interface. + +**Admin user**: The bootstrap program will also create an admin account in +PAUSE for you, with the username and password both set to the value of +`--username`. This is useful for testing, but if you're preparing a new PAUSE +instance, make sure you delete it. + +**certbot**: By default, certbot will generate a real, trusted certificate for +the new VM. That's useful to test with your web browser without security +alerts. The down side is that you can only generate a limited number of Let's +Encrypt certificates before being rate limited. Pass `--certbot-staging` to +use the staging certbot server. This is a real certificate, but isn't trusted +by default in common certificate roots, so your browser will complain. On the +other hand, you can generate quite a lot of them without being rate-limited. + +**plenv**: The bootstrap process will default to building perl from source and +installing all of PAUSE's depenedencies using `cpanm`. On our usual VM size, +this takes around ten minutes. It's useful to save a copy of the +`~pause/.plenv` as an archive file so avoid waiting on build and install. Put +that archive, as a `tar.bz2` file, somewhere on the web, and then provide the +URL to it as the `--plenv-url` option. As of 2024-04-26, a workable archive +can be found at https://dot-plenv.nyc3.digitaloceanspaces.com/dot-plenv.tar.bz2 + diff --git a/bootstrap/import-pause-data b/bootstrap/import-pause-data new file mode 100755 index 000000000..a622579f9 --- /dev/null +++ b/bootstrap/import-pause-data @@ -0,0 +1,49 @@ +#!/bin/bash + +# This program is meant to be run on a just-installed bootstrapped PAUSE host. +# It expects that you've gotten MySQL dumps from ../cron/mysql-dump.pl, +# preferably using --extended-insert, and put them in the current working +# directory. It will stop indexing, import all the files from PAUSE, and then +# also all the database content. Then it starts things back up. This can be +# used for testing operations on "real" data, but is also a useful step in +# transitioning to a new PAUSE install in production. +if [ $UID != "0" ]; then + echo "import-mod-db is meant to be run as root" >&2 + exit 1; +fi + +if [ ! -e "moddump.current.bz2" -o ! -e "authen_pausedump.current.bz2" ]; then + # The code used to do this: + # rsync -vaP pause.perl.org::pausedata/moddump.current.bz2 . + # + # ...but this only gets the moddump, and really we are going to need authen + # also. + echo "both moddump.current.bz2 and authen_pausedump.current.bz2 must be in cwd" >&2 + exit 1; +fi + +touch /etc/PAUSE.CLOSED + +systemctl stop paused + +sudo -u pause rsync --progress -av pause.perl.org::PAUSE/ /data/pause/pub/PAUSE/ + +echo 'SET GLOBAL innodb_flush_log_at_trx_commit=2' | mysql + +bunzip2 moddump.current.bz2 + +pv moddump.current | \ + perl -ple 's/^CHANGE MASTER.*//' | \ + sudo mysql mod + +bunzip2 authen_pausedump.current.bz2 + +pv authen_pausedump.current | \ + perl -ple 's/^CHANGE MASTER.*//' | \ + sudo mysql authen_pause + +echo 'SET GLOBAL innodb_flush_log_at_trx_commit=1' | mysql + +systemctl start paused + +rm /etc/PAUSE.CLOSED diff --git a/bootstrap/lib/Dobby/Client.pm b/bootstrap/lib/Dobby/Client.pm new file mode 100644 index 000000000..4998e7f9e --- /dev/null +++ b/bootstrap/lib/Dobby/Client.pm @@ -0,0 +1,315 @@ +use v5.32.0; +use warnings; + +package Dobby::Client; + +use experimental 'signatures'; +use parent 'IO::Async::Notifier'; + +use Carp (); +use Future::AsyncAwait; +use Future::Utils qw(repeat); +use IO::Async::Loop; +use JSON::MaybeXS; +use Net::Async::HTTP; + +sub configure ($self, %param) { + my @missing; + + KEY: for my $key (qw( bearer_token )) { + unless (defined $param{$key}) { + push @missing, $key; + next KEY; + } + + $self->{"__dobby_$key"} = $param{$key}; + } + + if (@missing) { + Carp::confess("missing required Dobby::Client parameters: @missing"); + } + + return; +} + +sub api_base { + return 'https://api.digitalocean.com/v2'; +} + +sub bearer_token { $_[0]{__dobby_bearer_token} } + +sub http ($self) { + return $self->{__dobby_http} //= do { + my $http = Net::Async::HTTP->new( + user_agent => 'Dobby/0', + ); + + $self->loop->add($http); + + $http; + }; +} + +async sub json_get ($self, $path) { + my $res = await $self->http->do_request( + method => 'GET', + uri => $self->api_base . $path, + headers => { + 'Authorization' => "Bearer " . $self->bearer_token, + }, + ); + + unless ($res->is_success) { + die "error getting $path at DigitalOcean: " . $res->as_string; + } + + my $json = $res->decoded_content(charset => undef); + decode_json($json); +} + +async sub json_get_pages_of ($self, $path, $key) { + my $url = $self->api_base . $path; + + my @items; + + while ($url) { + my $res = await $self->http->do_request( + method => 'GET', + uri => $url, + headers => { + 'Authorization' => "Bearer " . $self->bearer_token, + }, + ); + + unless ($res->is_success) { + die "error getting $path at DigitalOcean: " . $res->as_string; + } + + my $json = $res->decoded_content(charset => undef); + my $data = decode_json($json); + + die "no entry for $key in returned page" + unless exists $data->{$key}; + + push @items, $data->{$key}->@*; + $url = $data->{links}{pages}{next}; + } + + return \@items; +} + +async sub _json_req_with_body ($self, $method, $path, $payload) { + my $res = await $self->http->do_request( + method => $method, + uri => $self->api_base . $path, + headers => { + 'Authorization' => "Bearer " . $self->bearer_token, + }, + + content_type => 'application/json', + content => encode_json($payload), + ); + + unless ($res->is_success) { + die "error making $method to $path at DigitalOcean: " . $res->as_string; + } + + my $json = $res->decoded_content(charset => undef); + decode_json($json); +} + +async sub json_post ($self, $path, $payload) { + await $self->_json_req_with_body('POST', $path, $payload); +} + +async sub json_put ($self, $path, $payload) { + await $self->_json_req_with_body('PUT', $path, $payload); +} + +async sub delete_url ($self, $path) { + my $res = await $self->http->do_request( + method => 'DELETE', + uri => $self->api_base . $path, + headers => { + 'Authorization' => "Bearer " . $self->bearer_token, + }, + ); + + unless ($res->is_success) { + die "error deleting resource at $path in DigitalOcean: " . $res->as_string; + } + + return; +} + +async sub create_droplet ($self, $arg) { + state @required_keys = qw( name region size tags image ssh_keys ); + + my @missing; + KEY: for my $key (@required_keys) { + unless (defined $arg->{$key}) { + push @missing, $key; + next KEY; + } + } + + if (@missing) { + Carp::confess("missing required Dobby::Client parameters: @missing"); + } + + my $create_res = await $self->json_post( + "/droplets", + { + $arg->%{ @required_keys }, + }, + ); + + my $droplet = $create_res->{droplet}; + + unless ($droplet) { + Carp::confess("Error creating Droplet."); + } + + my $action_id = $create_res->{links}{actions}[0]{id}; + + unless (defined $action_id) { + Carp::confess( + "no action id from droplet action: " . encode_json($create_res) + ); + } + + my $waited = await $self->_do_action_status_f("/actions/$action_id"); + + return $droplet; +} + +async sub take_droplet_action ($self, $droplet_id, $action, $payload = {}) { + my $action_res = await $self->json_post("/droplets/$droplet_id/actions", { + %$payload, + type => $action, + }); + + my $action_id = $action_res->{action}{id}; + + unless (defined $action_id) { + Carp::confess( + "no action id from droplet action: " . encode_json($action_res) + ); + } + + await $self->_do_action_status_f("/droplets/$droplet_id/actions/$action_id"); + + return; +} + +async sub destroy_droplet ($self, $droplet_id) { + my $delete_res = await $self->delete_url("/droplets/$droplet_id"); + return; +} + +async sub _do_action_status_f ($self, $action_url) { + TRY: while (1) { + my $action = await $self->json_get($action_url); + my $status = $action->{action}{status}; + + # ugh, DO is now sometimes returning empty string in the status field + # -- michael, 2021-04-16 + $status = 'completed' if ! $status && $action->{action}{completed_at}; + + if ($status eq 'in-progress') { + await $self->loop->delay_future(after => 5); + next TRY; + } + + if ($status eq 'completed') { + return; + } + + if ($status eq 'errored') { + Carp::confess("action $action_url failed: " . encode_json($action->{action})); + } + + Carp::confess("action $action_url in unknown state: $status"); + } +} + +async sub _get_droplets ($self, $arg = {}) { + my $path = '/droplets?per_page=200'; + $path .= "&tag_name=$arg->{tag}" if $arg->{tag}; + + my $droplets_data = await $self->json_get($path); + + # TODO Obviously, this should lazily fetch etc. + if ($droplets_data->{links}{pages}{forward_links}) { + Carp::cluck("Single-page fetch did not find all droplets!"); + } + + unless ($droplets_data->{droplets}) { + Carp::cluck( + "getting /droplets didn't supply droplets: " . encode_json($droplets_data) + ); + } + + return $droplets_data->{droplets}->@*; +} + +async sub get_all_droplets ($self) { + await $self->_get_droplets; +} + +async sub get_droplets_with_tag ($self, $tag) { + await $self->_get_droplets({ tag => $tag }); +} + +async sub add_droplet_to_project ($self, $droplet_id, $project_id) { + my $path = "/projects/$project_id/resources"; + + await $self->json_post($path, { + resources => [ "do:droplet:$droplet_id" ], + }); +} + +async sub get_all_domain_records_for_domain ($self, $domain) { + my $path = '/domains/' . $domain . '/records'; + + # TODO Obviously, this should lazily fetch etc. + my $record_res = await $self->json_get("$path?per_page=200"); + return unless $record_res->{domain_records}; + return $record_res->{domain_records}->@*; +} + +async sub remove_domain_records_for_ip ($self, $domain, $ip) { + my $path = '/domains/' . $domain . '/records'; + + my @records = await $self->get_all_domain_records_for_domain($domain); + my @to_delete = grep {; $_->{data} eq $ip } @records; + my @deletions = map {; $self->delete_url("$path/$_->{id}") } @to_delete; + + return await Future->wait_all(@deletions); +} + +async sub point_domain_record_at_ip ($self, $domain, $name, $ip) { + my $path = '/domains/' . $domain . '/records'; + + my @records = await $self->get_all_domain_records_for_domain($domain); + + my (@existing) = grep {; $_->{name} eq $name && $_->{type} eq 'A' } @records; + + if (@existing) { + my @to_update = map {; $self->json_put("$path/$_->{id}", { data => $ip }) } + @existing; + await Future->wait_all(@to_update); + return; + } + + await $self->json_post($path, { + type => 'A', + name => $name, + data => $ip, + ttl => 30, + }); + + return; +} + +1; diff --git a/bootstrap/mkpause b/bootstrap/mkpause new file mode 100755 index 000000000..25e30fc3f --- /dev/null +++ b/bootstrap/mkpause @@ -0,0 +1,368 @@ +#!/usr/bin/env perl + +# If you're building a test PAUSE to test this automation, this is the place to +# start. This program requires a DigitalOcean API token, which is used to +# create a VM. This program then copies `selfconfig-root` to that VM and runs +# it. `selfconfig-root` will install packages, create users, and configure +# services. One of the created users is `pause`, which will be used to run the +# `selfconfig-pause` program found in this directory. +# +# For higher-level and more detailed view, check out the README in this +# directory. + +use v5.36.0; + +use lib 'lib'; + +use Dobby::Client; +use Future::AsyncAwait; +use Getopt::Long::Descriptive; +use IO::Async::Loop; +use Log::Dispatchouli; +use YAML::XS qw(LoadFile); + +my %default; +my $file = $ENV{PAUSE_MKPAUSE_CONFIG} // ".mkpause"; +if (-e $file) { + my ($config) = LoadFile($file); + %default = %$config; +} + +my ($opt, $usage) = describe_options( + '%c %o', + [ 'username|u=s', "your username; defaults to $ENV{USER}", + { default => $default{username} // $ENV{USER} // die 'no USER env var!' } ], + + # For serious business, we like c-16. -- rjbs, 2024-04-19 + [ 'size=s', "slug for DigitalOcean droplet", + { default => $default{size} // 'g-4vcpu-16gb' } ], + [ 'project_id=s', "if given, a DigitalOcean project id to add the VM to", + { default => $default{'project-id'} } ], + + [ 'box-ident|i=s', "identifying part of box name; defaults to --username" ], + [], + [ 'plenv-url|P=s', "URL to a tar.bz2 file to use for plenv (it's a shortcut)", + { default => $default{'plenv-url'} } ], + [], + [ 'certbot-staging|C!', 'use the staging version of certbot', + { default => $default{'certbot-staging'} } ], + [ 'enable-mail|m', "set up postfix config for outbound mail" ], + [ 'relay-host=s', "relay host for smtp" ], + [ 'relay-port=s', "relay port for smtp" ], + [ 'relay-username=s', "relay sasl username for smtp" ], + [ 'relay-password=s', "relay sasl password for smtp" ], + + [], + [ 'mode', 'hidden' => { + default => 'create', + one_of => [ + [ 'create', 'create the box if it does not exist' ], + [ 'list|l', 'list boxes that are running then quit' ], + [ 'destroy', 'destroy the box if it does exist' ], + ], + } + ], +); + +my sub ip_addr_for ($droplet) { + my ($ip_addr) = map { $_->{ip_address} } grep { $_->{type} eq 'public'} + $droplet->{networks}{v4}->@*; + + return $ip_addr; +} + +my $Logger = Log::Dispatchouli->new({ + facility => undef, + ident => 'unpause-boxo', + log_pid => 0, + to_stdout => 1, +}); + +my $loop = IO::Async::Loop->new; + +my $TOKEN = $ENV{DO_TOKEN} // $default{'api-token'} // die "no DigitalOcean API token\n"; + +my $dobby = Dobby::Client->new( + bearer_token => $TOKEN, +); + +$loop->add($dobby); + +if ($opt->list) { + my @droplets = $dobby->get_droplets_with_tag('unpause')->get; + + if (@droplets) { + $Logger->log("Droplets:"); + + my %drops = map {; $_->{name} => ip_addr_for($_) } @droplets; + + for my $k (sort keys %drops) { + $Logger->log([" %-20s %s", $k, $drops{$k}]); + } + } else { + $Logger->log("No unpause droplets found"); + } + + exit; +} + +my @mopts = qw(relay_host relay_port relay_username relay_password); + +if ($opt->enable_mail) { + my @m; + + for my $setting (@mopts) { + push @m, $setting unless $opt->$setting; + } + + $_ =~ s/_/-/g for @m; + + die "--enable-mail requires @m\n" if @m; +} + +my $domain = "fastmail.dev"; +my $username = $opt->username; +my $boxname = ($opt->box_ident // $username) . ".unpause"; + +my $todo = __PACKAGE__->can("do_" . $opt->mode); + +# This "can't happen". -- rjbs, 2024-03-23 +die "WTF: unknown mode of operation request" unless $todo; + +await $todo->(); + +#---( cut here )--- + +async sub do_create { + { + my @droplets = await $dobby->get_droplets_with_tag('unpause'); + my ($droplet) = grep {; $_->{name} eq $boxname } @droplets; + + if ($droplet) { + my $ip = ip_addr_for($droplet); + + my $extra = $ip ? " at root\@$ip" : ""; + + die "box already exists$extra\n"; + } + } + + my @key_ids; + { + my %want_key = map {; $_ => 1 } qw( matthew rjbs ); + my $keys = await $dobby->json_get_pages_of("/account/keys", 'ssh_keys'); + + my (@keys) = grep {; $want_key{$_->{name}} } @$keys; + + unless (@keys) { + die "can't find ssh keys to use!\n"; + } + + @key_ids = map {; $_->{id} } @keys; + } + + my $image = 'debian-12-x64'; + my $region = 'nyc3'; + my $size = $opt->size; + + my %droplet_create_args = ( + name => $boxname, + image => $image, + region => $region, + size => $size, + ssh_keys => \@key_ids, + tags => [ 'unpause' ], + ); + + $Logger->log([ "Creating droplet: %s", \%droplet_create_args ]); + + my $droplet = await $dobby->create_droplet(\%droplet_create_args); + + unless ($droplet) { + die "There was an error creating the box. Try again.\n"; + } + + # At this point, the box exists, but isn't quite up. The above result, for + # example, has no networks entry. We need to re-get it. + $Logger->log([ "Created droplet %i, now waiting for network...", $droplet->{id} ]); + + # We delay this because a completed droplet sometimes does not show up in GET + # /droplets immediately, which causes annoying problems. Waiting is a + # silly fix, but seems to work, and it's not like box creation is + # lightning-fast anyway. + await $loop->delay_future(after => 5); + + { + my $payload = await $dobby->json_get("/droplets/$droplet->{id}"); + $droplet = $payload->{droplet}; + } + + unless ($droplet) { + die "Box was created, but now I can't find it! Check the DigitalOcean console and maybe try again.\n"; + } + + my $ip_addr = ip_addr_for($droplet); + + $Logger->log([ "Droplet is now up on %s...", $ip_addr ]); + + if ($opt->project_id) { + await $dobby->add_droplet_to_project($droplet->{id}, $opt->project_id); + } + + $Logger->log("updating DNS names for $boxname"); + + await $dobby->point_domain_record_at_ip( + $domain, + "$boxname", + $ip_addr, + ); + + $Logger->log("Waiting for ssh to become available..."); + + my $ssh_up = await wait_for_port($ip_addr, 22); + + unless ($ssh_up) { + $Logger->log("The droplet was created, but ssh didn't come up. Your turn!"); + exit 1; + } + + $Logger->log("ssh is now available to $boxname.$domain ($ip_addr)"); + + sleep 1; + + $Logger->log("Now turning the box into a PAUSE server..."); + + system( + qw( + scp + -o UserKnownHostsFile=/dev/null + -o UpdateHostKeys=no + -o StrictHostKeyChecking=no + + selfconfig-root + ), + "root\@$ip_addr:", + ); + + my %mailopts; + + if ($opt->enable_mail) { + $mailopts{"--" . ($_ =~ s/_/-/gr)} = $opt->$_ for @mopts; + } + + system( + qw( + ssh + -o UserKnownHostsFile=/dev/null + -o UpdateHostKeys=no + -o StrictHostKeyChecking=no + + -l root + ), + $ip_addr, + qw( perl selfconfig-root ), + '--host', "$boxname.$domain", + '--user', $username, + '--pass', $username, # XXX: Do something better later. + ($opt->certbot_staging ? '--certbot-staging' : ()), + ($opt->enable_mail ? ('--enable-mail', %mailopts) : ()), + + ($opt->plenv_url ? ('--plenv-url' => $opt->plenv_url) : ()), + ); + + $Logger->log(<<~EOF); + + Done! If all went well, pause should be at: + + ssh root\@$ip_addr + ssh root\@$boxname.$domain + + https://$boxname.$domain + EOF +} + +async sub do_destroy { + my $droplet; + { + my @droplets = await $dobby->get_droplets_with_tag('unpause'); + ($droplet) = grep {; $_->{name} eq $boxname } @droplets; + } + + unless ($droplet) { + die "The box $boxname does not exist, and so cannot be destroyed.\n"; + } + + my $ip_addr = ip_addr_for($droplet); + + await $dobby->remove_domain_records_for_ip($domain, $ip_addr); + + $Logger->log([ "Destroying droplet: %s (%s)", $droplet->{id}, $droplet->{name} ]); + + await $dobby->destroy_droplet($droplet->{id}); + + $Logger->log([ "Destroyed droplet: %s", $droplet->{id} ]); +} + +async sub wait_for_port ($ip_addr, $port) { + my $max_tries = 20; + TRY: for my $try (1..$max_tries) { + my $done = eval { + my $socket; + + $socket = await $loop->connect(addr => { + family => 'inet', + socktype => 'stream', + port => 22, + ip => $ip_addr, + }); + + if ($socket) { + $Logger->log("We connected... let's see if we get a banner..."); + + $socket->blocking(1); + + local $SIG{ALRM} = sub { die "timed out\n" }; + alarm 5; + + my $banner; + $socket->recv($banner, 1024); + + alarm 0; + + if ($banner && $banner =~ /^SSH/) { + $Logger->log("Yup!"); + + return 1; + } + + $Logger->log("Nope, not yet."); + + return 0; + } + }; + + alarm 0; + + return 1 if $done; + + my $error = $@; + + if ($error && $error !~ /(Connection refused|timed out)/) { + $Logger->log([ + "weird error connecting to %s:22: %s", + $ip_addr, + $error, + ]); + } + + $Logger->log([ + "ssh on %s is not up, maybe wait and try again; %s tries remain", + $ip_addr, + $max_tries - $try, + ]); + + await $loop->delay_future(after => 1); + } + + return; +} diff --git a/bootstrap/selfconfig-pause b/bootstrap/selfconfig-pause new file mode 100755 index 000000000..d352379a6 --- /dev/null +++ b/bootstrap/selfconfig-pause @@ -0,0 +1,170 @@ +#!/usr/bin/perl + +# Normally, you won't run this program by hand. Instead, it's run indirectly +# by mkpause, which is run by a human. mkpause will create a new VM, copy +# selfconfig-root to it, and remotely run that program as root. +# selfconfig-root, in turn, will create a low-privilege user (pause) and run +# *this* program on the VM. +# +# This program's job is to install the Perl environment needed for PAUSE +# and then to configure PAUSE for operation on this VM. +# +# For higher-level and more detailed view, check out the README in this +# directory. + +use v5.36.0; + +use Carp qw(croak); +use Getopt::Long::Descriptive; +use Path::Tiny; + +my ($opt, $usage) = describe_options( + '%c %o', + [ 'authuser-pw=s', "password for auth user", { required => 1 } ], + [ 'moduser-pw=s', "password for mod user", { required => 1 } ], +); + +if (-e '/tmp/plenv-tarball.tar.bz2') { + chdir("/home/pause") or die "can't chdir to ~pause: $!"; + + # Somebody helpfully gave us a plenv directory to start from! Let's use it. + run_cmd(qw(tar jxvf /tmp/plenv-tarball.tar.bz2)); + + path("/home/pause/.bash_profile")->append(<<~'END'); + export PATH="$HOME/.plenv/bin:$PATH" + eval "$(plenv init -)" + END +} else { + chdir("/home/pause/pause") or die "can't chdir to ~pause/pause: $!"; + + # install plenv so we can manage a local perl version + run_cmd(qw( + git clone https://github.com/tokuhirom/plenv.git /home/pause/.plenv + )); + + path("/home/pause/.bash_profile")->append(<<~'END'); + export PATH="$HOME/.plenv/bin:$PATH" + eval "$(plenv init -)" + END + + # install perl-build so we can build a new perl + run_cmd(qw( + git clone https://github.com/tokuhirom/Perl-Build.git + /home/pause/.plenv/plugins/perl-build/ + )); + + run_cmd(qw( /home/pause/.plenv/bin/plenv install 5.36.0 -j 16 --noman )); + run_cmd(qw( /home/pause/.plenv/bin/plenv global 5.36.0 )); + + # install cpanm for perl dep management + run_cmd(qw( /home/pause/.plenv/bin/plenv install-cpanm )); + + # Pin DBD::mysql to 4.050. Newer doesn't work for some reason + # last I knew (mariadb problems?). And 4.052 breaks reconnecting + # after calls to disconnect, so we can't use that. 4.050 is what + # is in pause2 right now, so this should be fine. + run_cmd(qw( /home/pause/.plenv/shims/cpanm -n DBD::mysql@4.050 )); + + run_cmd(qw( /home/pause/.plenv/shims/cpanm -n --installdeps . )); +} + +chdir("/home/pause/pause") or die "can't chdir to ~pause/pause: $!"; + +for my $dir (qw( + /data/pause/ftp + /data/pause/pub + /data/pause/incoming + /data/pause/tmp +)) { + Path::Tiny::path($dir)->mkdir; +} + +# Set up pause config +for my $path (qw( + /data/pause/pub/PAUSE/authors/id + /data/pause/pub/PAUSE/modules + /data/pause/pub/PAUSE/PAUSE-git + + /home/pause/log + /home/pause/pause-private/lib + /home/pause/pid + /home/pause/run + /home/pause/testmail + /tmp/pause_1999 +)) { + path($path)->mkdir; +} + +chdir("/data/pause/pub/PAUSE/PAUSE-git") + || die "couldn't chdir to PAUSE-git: $!"; + +run_cmd(qw(git config --global user.email pause@pause.perl.org)); +run_cmd(qw(git config --global user.name), 'PAUSE Daemon'); +run_cmd(qw(git init --initial-branch master )); + +{ + # This imports a test key, which nobody should trust, and which has key id + # 6BA1716EFB099DB2. -- rjbs, 2024-04-25 + path("/home/pause/pause-private/gnupg-pause-batch-signing-home") + ->mkdir + ->chmod(0700); + + run_cmd(qw( + gpg + --homedir /home/pause/pause-private/gnupg-pause-batch-signing-home + --import + --armor /home/pause/pause/bootstrap/test-key.txt + )); +} + +my $config_file_contents = <<~'END'; + use strict; + package PAUSE; + + $ENV{EMAIL_SENDER_TRANSPORT} = 'Maildir'; + $ENV{EMAIL_SENDER_TRANSPORT_dir} = '/home/pause/testmail'; + + our $Config; + $Config->{AUTHEN_BACKUP_DIR} = "/home/pause/db-backup"; + + $Config->{AUTHEN_DATA_SOURCE_USER} = "authuser"; + $Config->{AUTHEN_DATA_SOURCE_PW} = "%%AUTHUSER_PW%%"; + + $Config->{MOD_DATA_SOURCE_USER} = "moduser"; + $Config->{MOD_DATA_SOURCE_PW} = "%%MODUSER_PW%%"; + + $Config->{MAIL_MAILER} = ["testfile"]; + $Config->{RUNDATA} = "/tmp/pause_1999"; + + $Config->{CHECKSUMS_SIGNING_PROGRAM} = "gpg"; + $Config->{CHECKSUMS_SIGNING_ARGS} = '--homedir /home/pause/pause-private/gnupg-pause-batch-signing-home --clearsign --default-key '; + $Config->{CHECKSUMS_SIGNING_KEY} = '6BA1716EFB099DB2'; + + $Config->{CRONPATH} = '/home/pause/pause/cron/'; + + $Config->{FTPPUB} = '/data/pause/pub/PAUSE/'; + $Config->{GITROOT} = '/data/pause/pub/PAUSE/PAUSE-git'; + $Config->{HTTP_ERRORLOG} = '/var/log/nginx/error.log'; + $Config->{INCOMING} = "file://data/pause/incoming/"; + $Config->{INCOMING_LOC} = '/data/pause/incoming'; + $Config->{MLROOT} = '/data/pause/pub/PAUSE/authors/id/'; + $Config->{ML_CHOWN_USER} = 'unsafe'; + $Config->{ML_CHOWN_GROUP} = 'unsafe'; + $Config->{ML_MIN_FILES} = 1; + $Config->{ML_MIN_INDEX_LINES} = 0; + $Config->{PAUSE_LOG} = "/home/pause/log/paused.log"; + $Config->{PAUSE_LOG_DIR} = "/home/pause/log/"; + $Config->{PAUSE_PUBLIC_DATA} = '/data/pause/pub/PAUSE/PAUSE-data'; + $Config->{PID_DIR} = "/home/pause/pid/"; + $Config->{TMP} = "/data/pause/tmp/"; + END + +$config_file_contents =~ s/%%AUTHUSER_PW%%/$opt->authuser_pw/e; +$config_file_contents =~ s/%%MODUSER_PW%%/$opt->moduser_pw/e; +path("/home/pause/pause-private/lib/PrivatePAUSE.pm")->spew($config_file_contents); + +sub run_cmd (@args) { + system {$args[0]} @args; + + croak "failed to run $args[0]" if $?; +} diff --git a/bootstrap/selfconfig-root b/bootstrap/selfconfig-root new file mode 100755 index 000000000..793404f07 --- /dev/null +++ b/bootstrap/selfconfig-root @@ -0,0 +1,425 @@ +#!/usr/bin/perl + +# Normally, you won't run this program by hand. Instead, it's run by mkpause, +# which is run by a human. mkpause will create a new VM, copy this program to +# it, and then remotely execute this program as root. +# +# This program's job is to set up the remote machine to become a PAUSE. It +# installs the needed packages, creates needed unix accounts, sets up and +# starts services, and also runs the selfconfig-pause program. +# selfconfig-pause is found right next to this selfconfig-root in the PAUSE +# repository, and runs as the pause user. +# +# For higher-level and more detailed view, check out the README in this +# directory. + +use v5.36.0; +use warnings; + +use Carp qw(croak); + +# If we don't have a term debconf gets angry +$ENV{TERM} //= 'xterm'; + +# On DigitalOcean, the journal won't be journaling at startup. Why? Well, I +# want to swear and say "because systemd!" but there seems to be an interesting +# reason, related to the machine id being baked into the image and then not +# matching that on the new cloud instance. I don't quite follow it. +# +# References: +# * https://unix.stackexchange.com/a/538881 +# * https://serverfault.com/a/1058260 +# +# Since we won't (??) be using DO for this in real work, I'm not trying to +# really fix it, I just want it logging before we start doing work. This will +# do the trick: +run_cmd(qw( systemctl restart systemd-journald.service )); + +# Don't run apt-get update if apt is already busy. We need to wait or we'll +# fail to update. Also wait for /var/lib/dpkg/lock-frontend +for my $try (1..30) { + system( + "fuser /var/lib/apt/lists/lock >/dev/null 2>/dev/null" + ); + + my $exit = $? >> 8; + last if $exit; + + warn "apt running, waiting 1s, try $try/30\n"; + + sleep 1; +} + +run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 update)); + +# Install system deps: +# +# Note that rjbs has been somewhat obnoxiously clever, below. Here, we install +# libpath-tiny-perl. That's Path::Tiny. Later in this very program, we will +# load and use this module. What a yutz. -- rjbs, 2024-03-23 +# +# Same goes for libgetopt-long-descriptive-perl. Yutz and proud. +# -- rjbs, 2024-04-05 +my @required_debs = qw( + build-essential + certbot + git + libdb-dev + libexpat1-dev + libgetopt-long-descriptive-perl + libpath-tiny-perl + libssl-dev + nginx + python3-certbot-nginx + unzip + zlib1g-dev + libsasl2-modules + ufw +); + +run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y), @required_debs); + +# Some packages we just don't want. +my @unwanted_debs = qw( + firewalld +); + +run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 remove -y), @unwanted_debs); + +require Getopt::Long::Descriptive; + +my ($opt, $usage) = Getopt::Long::Descriptive::describe_options( + '%c %o', + [ "host=s", "the hostname being used for this install", { required => 1 } ], + [ "user=s", "username for PAUSE admin to create", { required => 1 } ], + [ "pass=s", "password for PAUSE admin to create", { required => 1 } ], + [], + [ 'plenv-url=s', "curl-able URL to a tar.bz2 archive of a .plenv" ], + [], + [ 'certbot-staging|C', 'use the staging version of certbot' ], + [], + [ "enable-mail|m", "enable working postfix config", ], + [ 'relay-host=s', "relay host for smtp" ], + [ 'relay-port=s', "relay port for smtp" ], + [ 'relay-username=s', "relay sasl username for smtp" ], + [ 'relay-password=s', "relay sasl password for smtp" ], + [ 'volume-group=s', "volume group for data" ], +); + +my @mopts = qw(relay_host relay_port relay_username relay_password); + +if ($opt->enable_mail) { + my @m; + + for my $setting (@mopts) { + push @m, $setting unless $opt->$setting; + } + + $_ =~ s/_/-/g for @m; + + die "--enable-mail requires @m\n" if @m; +} + +my $hostname = $opt->host; +my $admin_user = uc $opt->user; +my $admin_pass = $opt->pass; + +# The --comment is here to suppress prompting for name, confirmation, etc. +run_cmd(qw(adduser pause --disabled-password --comment), 'PAUSE User'); +run_cmd(qw(adduser unsafe --disabled-password --comment), 'PAUSE Unsafe'); + +if ($opt->plenv_url) { + run_cmd('curl', $opt->plenv_url, '--output', '/tmp/plenv-tarball.tar.bz2'); +} + +require Path::Tiny; + +Path::Tiny::path("/data/mysql")->mkdir; + +Path::Tiny::path("/data/pause")->mkdir; +run_cmd("chown", "pause:", "/data/pause"); + +if (-e "/usr/sbin/lvcreate" && $opt->volume_group) { + my $vg = $opt->volume_group; + run_cmd(qw(lvcreate -L4G), qq($vg), qw(-n mysql)); + run_cmd(qw(lvcreate -L50G), qq($vg), qw(-n pause)); + Path::Tiny::path("/etc/fstab")->append(<<~EOF); +/dev/$vg/mysql /data/mysql ext4 defaults 0 2 +/dev/$vg/pause /data/pause ext4 defaults 0 2 +EOF + run_cmd(qw(systemctl daemon-reload)); + run_cmd(qw(mkfs.ext4 -j), qq(/dev/$vg/mysql)); + run_cmd(qw(mkfs.ext4 -j), qq(/dev/$vg/pause)); + run_cmd(qw(mount /data/mysql)); + run_cmd(qw(mount /data/pause)); + + # We have to chown *again* after mounting. We don't *only* chown here + # because we only enter this branch when --volume-group was passed! + run_cmd("chown", "pause:", "/data/pause"); +} + +# Partitioning! +Path::Tiny::path("/data/mysql/mysql")->mkdir; + +run_cmd(qw(ln -s /data/mysql/mysql /var/lib/mysql)); + +# Mariadb has to be installed _after_ partitioning. +run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y), + qw( + mariadb-server + default-libmysqlclient-dev + )); + +run_cmd(qw( + sudo -u pause git clone -b unpause https://git@github.com/rjbs/pause/ /home/pause/pause +)); + +# set up mysql databases and our pause user +run_cmd(qw(mysqladmin CREATE mod)); +run_sh('mysql mod < ~pause/pause/doc/mod.schema.txt'); + +run_cmd(qw(mysqladmin CREATE authen_pause)); +run_sh('mysql -u root authen_pause < ~pause/pause/doc/authen_pause.schema.txt'); + +run_cmd(qw(mysql mod -e), "INSERT INTO users (userid) VALUES ('$admin_user')"); + +my $crypted_pass = crypt $admin_pass, chr(rand(26)+97) . chr(rand(26)+97); + +run_cmd( + qw(mysql authen_pause -e), + "INSERT INTO usertable (user, password) VALUES ('$admin_user', '$crypted_pass')", +); + +run_cmd( + qw(mysql authen_pause -e), + "INSERT INTO grouptable (user, ugroup) VALUES ('$admin_user', 'admin')", +); + +my %db_password_for = ( + authuser => undef, + moduser => undef, +); + +{ + my sub rand_pw { + # Generates strings kinda like this one: b9l12-r5y9s-uc609-zey9q-61vjd + my @chars = (0..9, 'a' .. 'z'); + my $pw = join q{-}, + map {; join q{}, map {; $chars[ rand @chars ] } (1..5) } + (1..5); + + return $pw; + } + + for my $user (sort keys %db_password_for) { + $db_password_for{$user} = rand_pw(); + + run_cmd( + qw(mysql -e), + qq{CREATE USER $user IDENTIFIED BY '$db_password_for{$user}'}, + ); + } +} + +run_cmd( + qw(mysql -e), + q{GRANT DELETE, INDEX, INSERT, SELECT, UPDATE, LOCK TABLES ON `mod`.* TO 'moduser'@'%';}, +); + +run_cmd( + qw(mysql -e), + q{GRANT DELETE, INDEX, INSERT, SELECT, UPDATE, LOCK TABLES ON `authen_pause`.* TO 'authuser'@'%';}, +); + +run_cmd( + qw(mysql -e), + q{GRANT BINLOG MONITOR, RELOAD ON *.* TO 'moduser'@'%';}, +); + +run_cmd( + qw(mysql -e), + q{GRANT BINLOG MONITOR, RELOAD ON *.* TO 'authuser'@'%';}, +); + +my $nginx_config = <<~"END"; +# Set up nginx conf +upstream pause { + server 127.0.0.1:5000; +} + +server { + listen 80 default_server; + + location / { + proxy_pass http://pause; + proxy_set_header X-Forwarded-Host \$host; + proxy_set_header X-Forwarded-Server \$host; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto \$scheme; + + proxy_pass_request_headers on; + proxy_no_cache \$cookie_nocache \$arg_nocache\$arg_comment; + proxy_no_cache \$http_pragma \$http_authorization; + proxy_cache_bypass \$cookie_nocache \$arg_nocache \$arg_comment; + proxy_cache_bypass \$http_pragma \$http_authorization; + proxy_pass_header Authorization; + } + + server_name $hostname; +} +END + +Path::Tiny::path("/etc/nginx/sites-available/$hostname")->spew($nginx_config); + +unlink('/etc/nginx/sites-enabled/default'); +symlink("/etc/nginx/sites-available/$hostname", "/etc/nginx/sites-enabled/$hostname") + or die "can't symlink nginx conf: $!"; + +# Firewall config +run_cmd(qw(ufw allow http)); +run_cmd(qw(ufw allow https)); +run_cmd(qw(ufw allow rsync)); +run_cmd(qw(ufw allow ssh)); +run_cmd(qw(ufw --force enable)); + +# Install ssl cert +run_cmd( + qw(sudo certbot --nginx -d), + $hostname, + qw(--agree-tos -n --email pause@pause.perl.org), + + # This will use the staging server, which can be used to make lots more + # certificates that usual, but they aren't trusted. + ($opt->certbot_staging + ? ( qw( --server https://acme-staging-v02.api.letsencrypt.org/directory ) ) + : ()), +); + +Path::Tiny::path("/etc/rsyncd.conf")->spew(<<~'END'); + # cat rsyncd.conf + max connections = 12 + log file = /var/log/rsyncd + pid file = /var/run/PAUSE-rsyncd.pid + transfer logging = true + use chroot = true + timeout = 600 + + [PAUSE] + path = /data/pause/pub/PAUSE + + [authors] + path = /data/pause/pub/PAUSE/authors + + [modules] + path = /data/pause/pub/PAUSE/modules + + [scripts] + path = /data/pause/pub/PAUSE/scripts/new + + [pausedata] + path = /data/pause/pub/PAUSE/PAUSE-data + + [pausecode] + path = /data/pause/pub/PAUSE/PAUSE-code + + [pausegit] + path = /data/pause/pub/PAUSE/PAUSE-git + END + +run_cmd( + qw( sudo -u pause ), + "/home/pause/pause/bootstrap/selfconfig-pause", + "--authuser-pw", $db_password_for{authuser}, + "--moduser-pw", $db_password_for{moduser}, +); + +# XXX: I would like to not have or need this! -- rjbs, 2024-04-27 +run_cmd(qw(ln -s /data/pause /home/ftp)); + +Path::Tiny::path("/home/pause/pause/cron/CRONTAB.ROOT")->copy("/etc/cron.d/pause"); + +if ($opt->enable_mail) { + my $relayhost = $opt->relay_host; + my $relayport = $opt->relay_port; + + system( + q{echo postfix postfix/main_mailer_type select "Internet with smarthost" | debconf-set-selections} + ); + croak "failed to update debconf for main_mailer_type" if $?; + + system( + qq{echo postfix postfix/mailname string $hostname | debconf-set-selections} + ); + croak "failed to update debconf for mailname" if $?; + + system( + qq{echo postfix postfix/relayhost string \\[$relayhost\\]:$relayport | debconf-set-selections} + ); + croak "failed to update debconf for mailname" if $?; + + run_cmd(qw(apt-get -o DPkg::Lock::Timeout=60 install -y postfix)); + + my $cf = Path::Tiny::path("/home/pause/pause/etc/puppet/modules/pause/files/etc/postfix/main.cf-pause-us"); + + my $maincf = $cf->slurp_raw; + $maincf =~ s{daemon_directory = /usr/libexec/postfix}{daemon_directory = /usr/lib/postfix/sbin} + or warn "!!! Failed to replace daemon_directory !!!\n\n"; + $maincf =~ s{inet_interfaces = all}{inet_interfaces = localhost} + or warn "!!! Failed to replace inet_interfaces !!!\n\n"; + + $maincf .= <<~EOF; + relayhost = [$relayhost]:$relayport + + smtp_sasl_auth_enable = yes + smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd + smtp_sasl_security_options = + smtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crt + smtp_use_tls = yes + smtp_tls_wrappermode = yes + smtp_tls_security_level = encrypt + EOF + + Path::Tiny::path("/etc/postfix/main.cf")->spew_raw($maincf); + + my ($user, $pass) = ($opt->relay_username, $opt->relay_password); + + Path::Tiny::path("/etc/postfix/sasl_passwd")->spew_raw(<<~EOF); + [$relayhost]:$relayport $user:$pass + EOF + + run_cmd(qw(chmod 600 /etc/postfix/sasl_passwd)); + run_cmd(qw(postmap /etc/postfix/sasl_passwd)); + + run_cmd(qw( postfix stop )); + run_cmd(qw( postfix start )); +} + +for my $service (qw( paused pause-web )) { + Path::Tiny::path("/home/pause/pause/services/$service.service") + ->copy("/etc/systemd/system/$service.service"); + + run_cmd(qw( systemctl enable ), $service ); + run_cmd(qw( systemctl start ), $service ); + run_cmd(qw( systemctl status ), $service ); +} + +for my $service (qw( rsync )) { + run_cmd(qw( systemctl enable ), $service ); + run_cmd(qw( systemctl start ), $service ); + run_cmd(qw( systemctl status ), $service ); +} + +## SUBROUTINES GO DOWN BELOW +## THAT'S WHY THEY'RE CALLED SUBROUTINES + +sub run_cmd (@args) { + system {$args[0]} @args; + + croak "failed to run $args[0]" if $?; +} + +sub run_sh ($str) { + system $str; + + croak "failed to run shell command" if $?; +} diff --git a/bootstrap/test-key.txt b/bootstrap/test-key.txt new file mode 100644 index 000000000..d91256660 --- /dev/null +++ b/bootstrap/test-key.txt @@ -0,0 +1,81 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQVYBGYqZ6kBDADhY91ivv6YZ4pNswsL9iJDuwX/8Gge/G7nMNNRkxVlyRPpfUlt +23EDHW/FWj8fJqN/sHWoc8MavXE8A4mTe93x/5kmayjywbJJeDTJujcxCthdpzrL +8tFtD6GwmLnqdjxrJgYeTDfu5cy5upeocQFtd02Gy6J+z21x2EdxVHwjoO4cMvfQ +TvlQ2w6vkxK5cxKk3fXNoCd/+efaBVaPp2yjCXkMEGKIoyoJNXa7q5TL7tRotFUE +vSl00dIvvpxG/s7XRJbjuEfi4Hso5CI3y9gg3yFDC/aS5NsBjIRIEqMk5/5lYPw+ +bwP4fBzLbah3i/q10D+vpTaiU+ELcP04Nhe4pKhfSQEg2/+tX5OIXj0ZHN5hI04w +J09qbto3WolPnhW/Tmm0a9UQljNRcHs0nUTY+MFkwTN1XCLKVyYKjRWKrEtz6buO +Hj6bKDSIgPaZJQXBR0DktMdRbmDXepYtFu30DClj0NMGAwObcvzutZu09IgYbmLP +JNSp6USxU9d4vkcAEQEAAQAL+gIdDHLlR6FnXrQraK92Ao8FZrtHdJ87Ph0h3iFp +AcMByBOEoIPiCMOub3ty1/Niz0Zg7tR45bcRpFLd1Y0R/TihuRuHupAbJVm+55Kh +MxKWC1GI6CWNLf1xnHgkXUwnxvh7WrQJZdq5PS2QTOUhXKl9sGZS1GXR+k1SfIxr +7hwSAr+By/RimOXuZwjoLcIopFGArEiXDNj0tzqvhsIjFpE8Skz5DFwT+flIwxN2 +85a9cNthYOusnzr86+FJb6tOPTRDLaAt6vdi/L1YJAJMkOix7kIXfIn6UqJpc6ld +ugz9+C6CLrZUMhQeZa1prm+lIQ7/o/Pzg5t9MW7Nq/+0HSQMDgQ86UZDG2k5Unvi +cFX4B53qKVBHqVTMR01IhH9SFxj1kwOht6pn7xShy45liC3rZJCE/nXL8wyqNmmU +FWCtFTOWgJlHLpCLKRb4XlSfSpSUPxoOQMxYyUr3p0mUeZYwE7XHAFrNK98IkiVf +5eNqU0v05feegNL/vVxIzvqqaQYA5l2t5+a2XnF5JvGOkVUVwLcrW8+XQ2xbk2XA +Uc465+rMJEa/9dq7Mjd0hXCOcT9HMGqAFsIYHtbYuzQBzLfLdL0kRo3M6o6F6CAA +LCrB+P0Kd1MhxCgNHEGNA9lIK6ySUSjdeStesN5E54RjkKJxEFKAZci45dIwTgC/ +OE2gHSYab25IJkfRENMThlonf0ubUj1rXemYqG1QGgrH4PGybzNlHVOQj/VgEJQb +ozNjkjlLeaGoATUoHLV9CqtcKZRJBgD6eHD0O5RqG8eCtgTIaBO85SacIp0b/x67 +0mrQW/Z5+FlIVVnbAy94v9EPoXDz2cH+c+DM0X+W5VrjmZkNaKOh5i2U+EH4wMlo +D/gB+7ztuI0ceRV9DGd5eGe+XGLxSxCHIZn0TJi6d4mlFsJPQ0t95/StEvZ6cm24 +hA2/ApKGsHp4vqxAXLkGEbRES9X+Za+Txtg7WKWS0DrBKXfsUxA2Ws2rnPrcmrs9 +iw0C+cRGWCKTMXrMkVINyfeMBQNkng8F/1ZtF/afuQ5d2Uc0Q0QWaJJx3LvBLmBU +ycZdihxEwhtDhu4rDyEKVJt+Whu3NsjeLZQ8MNXh3sdTyWNsRX6N+ZCD2kRB4OuJ +d1aOgX9eVQKIABrvXE3IrOLArsHMUkjefa/qmIGRz5nXzQp+/6VtBwI1Gtqe6ahV +iUU4XPRuzbg7UJYwO+PsMVb0TQTUhne5HdTSyJDtFcIbAVGfWwP45/dOuPtKVBkX +EtUhW507zIkh6qIdauxo7z3KBqEg4p0+/dgRtBt0ZXN0aW5nLW9ubHlAcGF1c2Uu +cGVybC5vcmeJAdQEEwEKAD4WIQTpXI0CYrueC+0CKA5roXFu+wmdsgUCZipnqQIb +AwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBroXFu+wmdsr1PDACO +pEYS3uC3HYX88nurkEsQejOjGb3cNGPcMDwJ4ISUQp4upVR3o4QL9eBUwQEp4ERi +9zaLSFFoqhjZLtohrfZ9RFjgmKnYExnxbgNss2JuYnrB97cBt9CSVyemKBdQfoKu +/nwr8FejoZAdsOwth+FzvFX7/pIgylye8P4GQ/818NBIi/uTa3mz3t5mrAQdcPUY +z37rb5CiuOFZHYVPVTK/JqImcCXtuwk7aM3RU/L88WmWZf3Bl9G2k3YUY2cxAcVQ +eScs3an8E6XKCIe6OfEd4jXD3V9us0+77e7DpgHO4Hdtz7kbsgrN+odgij3/cjuX +3lXgjgeEdt6X15ilb4/vzs7c99XXXxqlPOFL3ArCxbQWexLggrS+muNvm4QxEE5+ +2BShEK5d4pBPc8hBUg40iQKyo5BJDCPqSHeVM9EZzcC6twa+ynhn5W9wu4iytLad +FpaUhXLbe4EuDsPKHUsY9gpkcsMzxfiL1iF1rzqiWJAcf2NRuE4ybwvMSCLzyGad +BVgEZipnqQEMANupLB1isvRXitZbZO+8fD01jAFkUXts+pfjmR/lmETk0DqXwThO +5irR+o0lymGxPIwMa5w2wcJC73r+L4l6VM296wkz7SArfUvcRGmPp2F4iWeca6i3 +IFDpgH6FsN+lLUWCugrvE78pPx2mWOyk3n4beizOe8IuH8iSnsqvm5OVEaH/66HE +9NWwVqC9HBSFBzyDqrb+suT0sitHPUPjJQh9nYi3JspblwcCrFLsykPGzSi7x/l2 +JMJbfQy2s/1w+PmT3t0NAB0LyABmsI+qZHjYiRqCIfgeeQPLmK2LcWyZ4y+sLcC/ +vwIPx5NPXZGSf6tjqkAE2ik7XXBP2294ozaWfiVYxgBwUdUBaX/deGz5a3s3WVPy +ARxexsiCgHtJm2/jgYdkBFusfT845xYj8ExwiunEeAQ3+ddbKlEg/7XK969bWdLP +xSjelyehsHuyg7YroT57ePX6RAV3LC+S/k0mRRMmdMetMcf1bbU4yZiJAg1zNbba +HKa/8XNkHvhpTwARAQABAAv9EKPeKFO5GXLQR7DLaAXapjW6PMNsEMeOrEGc8KJP +SK4fasY4PZsXK0NA5xRnzRgaGBnmGlgOPDh0ioKwTd/V23LPAqF2Yyt9kGjyncnj +dNTRbpHrmJpWzl+spj5bycq/hTaEAB61sOjfqSVYMy/CKGzNx+XUe5ShlRYrr/sM +SWp6LXraT3pAzzdOjgVawosuoyeY9R08/BngArtLftW5kSJtnqpmkTR4WnBEvFDH +zx6Y8sRW6npICpks/fDmmFpzE1wQG6Cqpsj5P/SnHqBBT86SpjhfdpfdZl+fXYJk +OAOsFdulscgiswXy5QpksI09fkCDdhVb9EL2lYUjjhZwLplzMk3jKA394J+yGx/t +kNGPhgKms15DOsrkZJigYJoM2cO6KpHQt6jSeVGIvOR3FWrQr/R27LowkHTj0R3e +QB8x3uL4IIfMisovVLheojJg0yfZFtWJN946M/E5JrWcTKa3yxJmGVCF6wvkExR1 +cGoz7tE8hfneHdrcSklKkKuBBgDkuHZnWgaF2/9muwtrvirnkCyXYCNyPJuh14tZ +FD5Ch3ya6MHonCIt6IZBt6kx2H7wpOAt8lS0C6K+pk/WpSLN18eiNkNwqvuy7USG +Xr8WNy1op+cM4LYE4+xlMNwVEqQ55TUEemBEnIgOaAZp/rX6lQEgnuRS/9A/AJVU +88JxDJL7ch111tpbyaF1vwNPIr5QyRX/WOG0qoFR9zrbscSp6fRoaCiTQlIEE6rC +99yGoaGiwg/4rVRH1ACQiRscYqEGAPXcFrHRUJc9iETM70IjKU3pQMxnlo3BNSrU +cHtBLKV0Mi55MQ5TKh/INYLSXUjV6f+ii/2cTKdCfU61Ex0bOx8y4X7IKaIaWjn+ +ALvs5g7sTSbDuvt+RQV+rSzZRmuFXyyc+7Y8w+F+VUPcJm8rLRdJH9UwFceBqTw4 +H3PJt+OgAVEa0qXI+csM8aBM3RmIows3N5u/NNUOEFkwRVjBnZ/q2tYDXjLFDFbI +DxvArQQr+UUO9bm4YUGd99b7oO017wYAqU0lUNrl+K/FNQo6XZDoWEXBCM//K+vA +E0PGtj+DZP/TFQ2hIlfZPEbHLIXc8Yq2qDz83z0rYd36LT1y4OvKlNQ9nppxzAxM +Svl06NZ7PUx2/Bql2loLQWfLPC/uY4teb167rx+TACfD/kOb8EXqyeYkjJNZeEWr +RPgfCi9yPXKTnaT1QdNOlf3ijPz/MbogZHk4wosC3PcFO1/tcABv79ovZJIBsLKR +EQOhjT+YaV53O+e6DVU1ArccaExL0cFl3+SJAbYEGAEKACAWIQTpXI0CYrueC+0C +KA5roXFu+wmdsgUCZipnqQIbDAAKCRBroXFu+wmdsm7JC/97EXNdkjPnJw7xIj+U +27NSisRKHtDKKlPJMPJ94Id1SKOwGECyTDUe3+51DvtyOVkmxHdOLal9QjdhCybI +SGJPklO2o+Af7WPmPdERcQZqLcxyGZJQe1s3zZ5MI/GNeu9Kci/bN0SITAeL1VOJ +V1BMgIhUVRtxITF95ypU05xi5BCrn2CmWnNWS1S3VthUY2Sp2DPO4KTlgH78gehP +VGUw7KrfNzsFiOEyzPDrwXpZe5Y9quwGnB8MFcapw59juN5bUZuJWPDEAbVozjDb +QCxEqhtXFZwsy+p8VWd8XfzyQ/Z6GDiuxb156FhRPbYfvqxrJVWqRSV5S3Om3QyG +zqufhduxDetIlj9/P37AuxZxxOFxtFb/lTN/qhs4vaWJ3NIiK414ok/DHi4WYm6R +cS+DRuARWktZ9KNlmyJ9/xDCgv3UT/x5QtXrwBeX97vOvC7EzVfbtlCc7TuA3Ay7 +h+EjflnLw/Nw84iZQKy1WTXN/kLEzZYK1ZTMNTIaE3Ps1Bo= +=dwTJ +-----END PGP PRIVATE KEY BLOCK----- diff --git a/cpanfile b/cpanfile index ed725cbd9..b0009040a 100644 --- a/cpanfile +++ b/cpanfile @@ -47,6 +47,7 @@ requires 'Plack::Middleware::ReverseProxy'; requires 'Plack::Middleware::ServerStatus::Tiny'; requires 'Process::Status'; requires 'Set::Crontab'; +requires 'Starman'; requires 'String::Random'; requires 'Text::Format'; requires 'Text::Markdown::Hoedown'; diff --git a/cron/CRONTAB.ROOT b/cron/CRONTAB.ROOT index d141001b2..4b4d68748 100644 --- a/cron/CRONTAB.ROOT +++ b/cron/CRONTAB.ROOT @@ -1,39 +1,38 @@ -MAILTO=andreas.koenig.5c1c1wmb@franz.ak.mind.de -PATH=/opt/perl/current/bin:/usr/bin:/bin:/home/puppet/pause/cron +# MAILTO=andreas.koenig.5c1c1wmb@franz.ak.mind.de -# HEADER: This file was autogenerated at Fri Dec 21 16:00:34 +0000 2012 by puppet. -# HEADER: While it can still be managed manually, it is definitely not recommended. -# HEADER: Note particularly that the comments starting with 'Puppet Name' should -# HEADER: not be deleted, as doing so could cause duplicate cron jobs. -# Puppet Name: puppet-restart -#6 4 * * * sleep $(expr $RANDOM \% 300); /sbin/service puppet restart > /dev/null +PATH=/home/pause/.plenv/shims:/usr/bin:/home/pause/pause/cron +PAUSE_REPO=/home/pause/pause +PAUSE_ROOT=/data/pause/pub/PAUSE -* * * * * recentfile-aggregate.sh -* * * * * date -u +"\%s \%a \%b \%e \%T \%Z \%Y" > /home/ftp/tmp/02STAMP && mv /home/ftp/tmp/02STAMP /home/ftp/pub/PAUSE/authors/02STAMP && /opt/perl/current/bin/perl -I /home/puppet/pause/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' /home/ftp/pub/PAUSE/authors/02STAMP -08 * * * * date -u +"\%s \%FT\%TZ" > /home/ftp/tmp/02STAMPm && mv /home/ftp/tmp/02STAMPm /home/ftp/pub/PAUSE/modules/02STAMP && /opt/perl/current/bin/perl -I /home/puppet/pause/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' /home/ftp/pub/PAUSE/modules/02STAMP -52 * * * * perl /home/puppet/pause/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log -04 7 * * 6 perl /home/puppet/pause/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --symlinkinventory -17,29,41,53 * * * * perl /home/puppet/pause/cron/mldistwatch --logfile /var/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite -12 06,14,22 * * * perl /home/puppet/pause/cron/update-checksums.pl -29 * * * * perl /home/puppet/pause/cron/cleanup-incoming.pl -*/3 * * * * perl /home/puppet/pause/cron/cleanup-apachecores.pl -59 * * * * perl /home/puppet/pause/cron/cron-daily.pl -37 05 * * * perl /home/puppet/pause/cron/gmls-lR.pl -47 07,13,19,01 * * * perl /home/puppet/pause/cron/mysql-dump.pl -19 * * * * perl /home/puppet/pause/cron/make-mirror-yaml.pl -26,56 * * * * perl /home/puppet/pause/cron/publish-crontab.pl -21 */6 * * * perl /home/puppet/pause/cron/rm_stale_links -23 07,13,19,01 * * * run_mirrors.sh -22 * * * * perl /home/puppet/pause/cron/sync-04pause.pl -03 07,13,18,01 * * * cd /home/ftp/pub/PAUSE/PAUSE-git && (git gc && git push -u origin master) >> /var/log/git-gc-push.out -4,11,19,26,34,42,49,56 * * * * zsh /home/puppet/pause/cron/assert-paused-running.zsh -18 * * * * perl /home/puppet/pause/cron/cron-p6daily.pl +## STUFF RJBS DID TO PUT THIS INTO UNPAUSE: +## * replace a bunch of paths: +## * /opt/perl/current/bin with /usr/bin/perl +## * put "perl" in front of things to use plenv perl instead of system perl +## * put the pause repo's cron directory in path *and use it* +## +## …and we will write this to /etc/cron.d/SOMETHING -#7 3,9 * * * perl /home/kstar/cron/indexscripts.pl >/dev/null 2>&1 -#7 2 * * 0 perl /home/kstar/cron/indexscripts.pl -f +# ??? +* * * * * pause $PAUSE_REPO/cron/recentfile-aggregate -#>19:47:02 root@pause2:/home/puppet/pause# PATH=/opt/perl/current/bin:/usr/bin:/bin:/home/puppet/pause/cron perl -Ilib -c bin/indexscripts.pl -#bin/indexscripts.pl syntax OK +# some kind of PAUSE heartbeat/health check system? +* * * * * pause date -u +"\%s \%a \%b \%e \%T \%Z \%Y" > /tmp/02STAMP && mv /tmp/02STAMP $PAUSE_ROOT/authors/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/authors/02STAMP +08 * * * * pause date -u +"\%s \%FT\%TZ" > /tmp/02STAMPm && mv /tmp/02STAMPm $PAUSE_ROOT/modules/02STAMP && perl -I $PAUSE_REPO/lib -e 'use PAUSE; PAUSE::newfile_hook(shift)' $PAUSE_ROOT/modules/02STAMP -46 0,6,12,18 * * * perl -I/home/puppet/pause/lib /home/puppet/pause/bin/indexscripts.pl >/home/puppet/pause/bin/indexscripts.pl.out 2>&1 -7 2 * * 0 perl -I/home/puppet/pause/lib /home/puppet/pause/bin/indexscripts.pl -f +# THE INDEXER +52 * * * * pause $PAUSE_REPO/cron/mldistwatch --logfile /home/pause/log/mldistwatch.cron.log +04 7 * * 6 pause $PAUSE_REPO/cron/mldistwatch --logfile /home/pause/log/mldistwatch.cron.log --symlinkinventory +17,29,41,53 * * * * pause $PAUSE_REPO/cron/mldistwatch --logfile /home/pause/log/mldistwatch.cron.log --fail-silently-on-concurrency-protection --rewrite + +12 06,14,22 * * * pause $PAUSE_REPO/cron/update-checksums.pl +29 * * * * pause $PAUSE_REPO/cron/cleanup-incoming.pl +59 * * * * pause $PAUSE_REPO/cron/cron-daily.pl +37 05 * * * pause $PAUSE_REPO/cron/gmls-lR.pl +47 07,13,19,01 * * * pause $PAUSE_REPO/cron/mysql-dump.pl +21 */6 * * * pause $PAUSE_REPO/cron/rm_stale_links +23 07,13,19,01 * * * pause $PAUSE_REPO/cron/run_mirrors.sh +22 * * * * pause $PAUSE_REPO/cron/sync-04pause.pl +10 09,15,21,03 * * * pause cd $PAUSE_ROOT/PAUSE-git && (git gc && git push -u origin master) >> /home/pause/log/git-gc-push.out +18 * * * * pause $PAUSE_REPO/cron/cron-p6daily.pl +46 0,6,12,18 * * * pause perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl > $PAUSE_REPO/bin/indexscripts.pl.out 2>&1 +7 2 * * 0 pause perl -I $PAUSE_REPO/lib $PAUSE_REPO/bin/indexscripts.pl -f diff --git a/cron/assert-paused-running.zsh b/cron/assert-paused-running.zsh deleted file mode 100755 index 362944134..000000000 --- a/cron/assert-paused-running.zsh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/zsh - -if ! /etc/init.d/PAUSE-paused status > /dev/null ; then - /etc/init.d/PAUSE-paused start -fi diff --git a/cron/cleanup-apachecores.pl b/cron/cleanup-apachecores.pl deleted file mode 100755 index ed18c90ea..000000000 --- a/cron/cleanup-apachecores.pl +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/perl - -# use 5.010; -use strict; -use warnings; - -=head1 NAME - - - -=head1 SYNOPSIS - - - -=head1 OPTIONS - -=over 8 - -=cut - -my @opt = <<'=back' =~ /B<--(\S+)>/g; - -=item B<--coredir=s> - -Directory to cleanup. Defaults to /opt/apache/cores. - -=item B<--filerx=s> - -Regular expression that filters files subject to the cleanup. Defaults to C - -=item B<--help|h!> - -This help - -=item B<--keep=i> - -Number of files to keep. Defaults to 10. - -=item B<--verbose!> - -Report about deletions. - -=back - -=head1 DESCRIPTION - - - -=cut - - -use FindBin; -use lib "$FindBin::Bin/../lib"; -BEGIN { - push @INC, qw( ); -} - -use Dumpvalue; -use File::Basename qw(dirname); -use File::Path qw(mkpath); -use File::Spec; -use File::Temp; -use Getopt::Long; -use Pod::Usage; -use Hash::Util qw(lock_keys); - -our %Opt; -lock_keys %Opt, map { /([^=|!]+)/ } @opt; -GetOptions(\%Opt, - @opt, - ) or pod2usage(1); - -$Opt{coredir} ||= "/opt/apache/cores"; -$Opt{keep} ||= 10; -$Opt{filerx} ||= q{^core\.\d+\z}; -my $filerxqr = qr{$Opt{filerx}}; -opendir my $dh, $Opt{coredir} or die "Could not open '$Opt{coredir}': $!"; -my @ls; -for my $dirent (readdir $dh) { - next unless $dirent =~ $filerxqr; - push @ls, "$Opt{coredir}/$dirent"; -} -my @ls_sorted_newest_first = map { $_->[0] } - sort { $a->[1] <=> $b->[1] } - map { [$_, -M $_] } - @ls; -while (@ls_sorted_newest_first > $Opt{keep}) { - my $unlink = pop @ls_sorted_newest_first; - print STDERR "About to delete '$unlink'..." if $Opt{verbose}; - unlink $unlink or die "Could not unlink '$unlink': $!"; - print STDERR "Done\n" if $Opt{verbose}; -} - -# Local Variables: -# mode: cperl -# cperl-indent-level: 4 -# End: diff --git a/cron/cleanup-incoming.pl b/cron/cleanup-incoming.pl index 3cde9fa37..d2233493f 100755 --- a/cron/cleanup-incoming.pl +++ b/cron/cleanup-incoming.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/home/pause/.plenv/shims/perl use FindBin; use lib "$FindBin::Bin/../lib"; diff --git a/cron/cron-daily.pl b/cron/cron-daily.pl index afba5c4bc..2d2f70419 100755 --- a/cron/cron-daily.pl +++ b/cron/cron-daily.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl use FindBin; use lib "$FindBin::Bin/../lib"; @@ -16,6 +16,7 @@ use Email::Sender::Simple (); use strict; +use warnings; use vars qw( $last_str $last_time $SUBJECT @listing $Dbh); # @@ -28,6 +29,8 @@ $TIME[5] += 1900; my $TIME = sprintf "%02d" x 5, @TIME[ 5, 4, 3, 2, 1 ]; +my $TMPFILE = "/tmp/00whois.new"; + my $zcat = $PAUSE::Config->{ZCAT_PATH}; die "no executable zcat" unless -x $zcat; my $gzip = $PAUSE::Config->{GZIP_PATH}; @@ -337,7 +340,7 @@ sub send_the_mail { header_str => [ Subject => $SUBJECT, To => $PAUSE::Config->{ADMIN}, - From => "cron daemon cron-daily.pl ", + From => "cron daemon cron-daily.pl ", ], body_str => join(q{}, @blurb), ); @@ -394,7 +397,7 @@ sub whois { scalar(gmtime), $0, ); - open FH, ">00whois.new" or return "Error: Can't open 00whois.new: $!"; + open FH, ">", $TMPFILE or return "Error: Can't open $TMPFILE: $!"; if ($] > 5.007) { require Encode; binmode FH, ":utf8"; @@ -574,10 +577,10 @@ sub whois { ]; close FH; - if (compare '00whois.new', "$PAUSE::Config->{MLROOT}/../00whois.html") { - report qq[copy 00whois.new $PAUSE::Config->{MLROOT}/../00whois.html\n\n]; + if (compare $TMPFILE, "$PAUSE::Config->{MLROOT}/../00whois.html") { + report qq[copy $TMPFILE $PAUSE::Config->{MLROOT}/../00whois.html\n\n]; my $whois_file = "$PAUSE::Config->{MLROOT}/../00whois.html"; - xcopy '00whois.new', $whois_file; + xcopy $TMPFILE, $whois_file; PAUSE::newfile_hook($whois_file); my $xml_file = "$PAUSE::Config->{MLROOT}/../00whois.xml"; diff --git a/cron/cron-p6daily.pl b/cron/cron-p6daily.pl index 48e1c3689..86d4cad15 100755 --- a/cron/cron-p6daily.pl +++ b/cron/cron-p6daily.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl use FindBin; use lib "$FindBin::Bin/../lib"; @@ -67,7 +67,7 @@ sub send_the_mail { header_str => [ Subject => $SUBJECT, To => $PAUSE::Config->{ADMIN}, - From => "cron daemon cron-p6daily.pl ", + From => "cron daemon cron-p6daily.pl ", ], body_str => join(q{}, @blurb), ); diff --git a/cron/gmls-lR.pl b/cron/gmls-lR.pl index 1af734d82..d107f13a9 100755 --- a/cron/gmls-lR.pl +++ b/cron/gmls-lR.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl =comment @@ -18,6 +18,7 @@ use PAUSE (); use File::Compare qw(compare); use strict; +use warnings; chdir $PAUSE::Config->{FTPPUB} or die "Could not chdir to $PAUSE::Config->{FTPPUB}: $!"; mkdir "indexes", 0755 unless -d "indexes"; diff --git a/cron/make-mirror-yaml.pl b/cron/make-mirror-yaml.pl deleted file mode 100755 index b75e38441..000000000 --- a/cron/make-mirror-yaml.pl +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/local/bin/perl - -use strict; -use warnings; -use CPAN::Indexer::Mirror 0.05; # atomic writes -use File::Path qw(mkpath); -use LWP::UserAgent; - -use FindBin; -use lib "$FindBin::Bin/../lib"; -use PAUSE; - -die "FTP_RUN not defined" unless defined $PAUSE::Config->{FTP_RUN}; -my $rundir = "$PAUSE::Config->{FTP_RUN}/mirroryaml"; -mkpath $rundir; -my $ua = LWP::UserAgent->new(agent => "PAUSE/20080922"); -my $resp = $ua->mirror($PAUSE::Config->{MIRRORED_BY_URL},"$rundir/MIRRORED.BY"); -die "Could not mirror: ".$resp->status_line unless $resp->is_success || 304 eq $resp->code; - -CPAN::Indexer::Mirror->new( - root => $rundir, - )->run; diff --git a/cron/mldistwatch b/cron/mldistwatch index b187f38ea..99e447b8a 100755 --- a/cron/mldistwatch +++ b/cron/mldistwatch @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/home/pause/.plenv/shims/perl =head1 NAME diff --git a/cron/mysql-dump.pl b/cron/mysql-dump.pl index 08fc1c42e..a5dcac459 100755 --- a/cron/mysql-dump.pl +++ b/cron/mysql-dump.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl =pod @@ -15,6 +15,7 @@ =cut use strict; +use warnings; use FindBin; use lib "$FindBin::Bin/../lib"; @@ -41,7 +42,6 @@ cfg_dsn => "MOD_DATA_SOURCE_NAME", cfg_user => "MOD_DATA_SOURCE_USER", cfg_pw => "MOD_DATA_SOURCE_PW", - master => 1, }, {backupdir => $PAUSE::Config->{AUTHEN_BACKUP_DIR}, cfg_dsn => "AUTHEN_DATA_SOURCE_NAME", diff --git a/cron/publish-crontab.pl b/cron/publish-crontab.pl index 96671171e..431966181 100755 --- a/cron/publish-crontab.pl +++ b/cron/publish-crontab.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/home/pause/.plenv/shims/perl die " # since we are switching to use /etc/cron.d/pause2016 for the core diff --git a/cron/recentfile-aggregate b/cron/recentfile-aggregate new file mode 100755 index 000000000..48dbe830f --- /dev/null +++ b/cron/recentfile-aggregate @@ -0,0 +1,25 @@ +#!/home/pause/.plenv/shims/perl +use v5.36.0; + +use FindBin; +use lib "$FindBin::Bin/../lib"; + +use PAUSE; +use Path::Tiny; + +my $PAUSE_ROOT = $PAUSE::Config->{FTPPUB}; + +die "FTPPUB directory ($PAUSE_ROOT) isn't there!\n" + unless -d $PAUSE_ROOT; + +my $root = path($PAUSE_ROOT); + +system( + '/home/pause/.plenv/shims/rrr-aggregate', + $root->child('authors/RECENT-1h.yaml'), +); + +system( + '/home/pause/.plenv/shims/rrr-aggregate', + $root->child('modules/RECENT-1h.yaml'), +); diff --git a/cron/recentfile-aggregate.sh b/cron/recentfile-aggregate.sh deleted file mode 100755 index 18d4eaffb..000000000 --- a/cron/recentfile-aggregate.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -export PATH=/opt/perl/current/bin:/usr/local/perl/bin:$PATH -rrr-aggregate /home/ftp/pub/PAUSE/authors/RECENT-1h.yaml -rrr-aggregate /home/ftp/pub/PAUSE/modules/RECENT-1h.yaml diff --git a/cron/restart-httpd b/cron/restart-httpd index ceeb716cf..e4ca87fd5 100755 --- a/cron/restart-httpd +++ b/cron/restart-httpd @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/home/pause/.plenv/shims/perl # scheduled for demise as soon as the pause2.develooper.com has taken over diff --git a/cron/rm_stale_links b/cron/rm_stale_links index dab51d0f2..e04f4f595 100755 --- a/cron/rm_stale_links +++ b/cron/rm_stale_links @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w -- -*- mode: cperl -*- +#!/home/pause/.plenv/shims/perl =pod @@ -18,16 +18,29 @@ use lib "$FindBin::Bin/../lib"; use PAUSE (); use strict; +use warnings; use File::Find; use File::Spec; chdir "$PAUSE::Config->{MLROOT}../.."; +my %KEEP_FOREVER = ( + # No matter what, do not delete these paths! For an example of why this is + # interesting, imagine a freshly installed PAUSE that has no files uploaded + # yet. Without protecting authors/id, it could be deleted. Later, something + # will try to upload authors/id/X/XY/XYZZY/Foo-1.23.tar.gz, but it can't, + # because authors/id has been removed. Possibly it should be willing to + # create the full tree, but right now, it can't. So, we will avoid deleting + # it instead. + 'authors/id' => 1, +); + find( { bydepth => 1, wanted => sub { return if /^\.\.?$/; + return if $KEEP_FOREVER{ $File::Find::name }; my($dev,$ino,$mode,$nlink,$uid,$gid) = lstat($_); if (-l $_ && ! -e $_){ warn "unlinking stale $File::Find::name\n"; diff --git a/cron/run_mirrors.sh b/cron/run_mirrors.sh index a6c73f190..b498f6ded 100755 --- a/cron/run_mirrors.sh +++ b/cron/run_mirrors.sh @@ -11,4 +11,4 @@ else exit 1 fi -perl /usr/bin/mirror -C$MIRRDIR/$MIRRCNF $MIRRDIR/mymirror.config +/home/pause/.plenv/shims/perl /usr/bin/mirror -C$MIRRDIR/$MIRRCNF $MIRRDIR/mymirror.config diff --git a/cron/sync-04pause.pl b/cron/sync-04pause.pl index f9e1197b5..200a194c4 100755 --- a/cron/sync-04pause.pl +++ b/cron/sync-04pause.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl +#!/home/pause/.plenv/shims/perl use FindBin; use lib "$FindBin::Bin/../lib"; diff --git a/cron/update-checksums.pl b/cron/update-checksums.pl index 7606a1a0b..f0b03cd38 100755 --- a/cron/update-checksums.pl +++ b/cron/update-checksums.pl @@ -1,4 +1,4 @@ -#!/usr/local/bin/perl -w +#!/home/pause/.plenv/shims/perl # use 5.010; use strict; @@ -60,7 +60,7 @@ =head1 DESCRIPTION use Hash::Util qw(lock_keys); use Pod::Usage; -my $lockfile = "/var/run/PAUSE-update-checksums.LCK"; +my $lockfile = "/home/pause/run/PAUSE-update-checksums.LCK"; use Fcntl qw( :flock :seek O_RDONLY O_RDWR O_CREAT ); my $lfh; unless (open $lfh, "+<", $lockfile) { diff --git a/lib/PAUSE.pm b/lib/PAUSE.pm index 0fb7c455b..abc31f836 100644 --- a/lib/PAUSE.pm +++ b/lib/PAUSE.pm @@ -123,7 +123,6 @@ $PAUSE::Config ||= CHECKSUMS_SIGNING_PROGRAM => ('gpg'), CHECKSUMS_SIGNING_ARGS => '--homedir /home/puppet/pause-private/gnupg-pause-batch-signing-home --clearsign --default-key ', CHECKSUMS_SIGNING_KEY => '450F89EC', - BATCH_SIG_HOME => "/home/puppet/pause-private/gnupg-pause-batch-signing-home", MIN_MTIME_CHECKSUMS => 1300000000, # invent a threshold for oldest mtime HAVE_PERLBAL => 1, ZCAT_PATH => (List::Util::first { -x $_ } ("/bin/zcat", "/usr/bin/zcat" )), diff --git a/services/pause-web.service b/services/pause-web.service new file mode 100644 index 000000000..92d842652 --- /dev/null +++ b/services/pause-web.service @@ -0,0 +1,25 @@ +[Unit] +Description=PAUSE web service +Documentation=https://github.com/andk/PAUSE/ + +# Wait for the network to be up +After=network-online.target + +# Require the network service to be present, and at least started at the same time as this service +Wants=network-online.target + +[Service] +Type=simple + +User=pause +Group=pause +WorkingDirectory=/home/pause/pause + +# Should not daemonize +ExecStart=/home/pause/.plenv/shims/plackup -s Starman --port 5000 app_2017.psgi + +Restart=on-failure +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/services/paused.service b/services/paused.service new file mode 100644 index 000000000..7ccf17334 --- /dev/null +++ b/services/paused.service @@ -0,0 +1,25 @@ +[Unit] +Description=PAUSE service +Documentation=https://github.com/andk/PAUSE/ + +# Wait for the network to be up +After=network-online.target + +# Require the network service to be present, and at least started at the same time as this service +Wants=network-online.target + +[Service] +Type=simple + +User=pause +Group=pause +WorkingDirectory=/home/pause/pause + +# Should not daemonize +ExecStart=/home/pause/.plenv/shims/perl /home/pause/pause/bin/paused --pidfile /home/pause/pid/paused.pid + +Restart=on-failure +RestartSec=30 + +[Install] +WantedBy=multi-user.target