From 4472844aceb9d456f244fad891fb742179670ba3 Mon Sep 17 00:00:00 2001 From: Saurov Chandra Biswas <48715910+souravbiswassanto@users.noreply.github.com> Date: Fri, 15 Mar 2024 21:14:43 +0600 Subject: [PATCH] Add support for postgres replication slot (#31) Signed-off-by: souravbiswassanto --- role_scripts/10/primary/start.sh | 9 ++++-- role_scripts/10/standby/ha_backup_job.sh | 10 ++++-- role_scripts/10/standby/run.sh | 33 ++++++++++++++++++-- role_scripts/10/standby/warm_stanby.sh | 9 ++++-- role_scripts/11/primary/start.sh | 9 ++++-- role_scripts/11/standby/ha_backup_job.sh | 9 ++++-- role_scripts/11/standby/run.sh | 32 +++++++++++++++++-- role_scripts/11/standby/warm_stanby.sh | 9 ++++-- role_scripts/12/primary/start.sh | 9 ++++-- role_scripts/12/standby/ha_backup_job.sh | 12 ++++++- role_scripts/12/standby/run.sh | 37 +++++++++++++++++++++- role_scripts/12/standby/warm_stanby.sh | 12 ++++++- role_scripts/13/primary/start.sh | 9 ++++-- role_scripts/13/standby/ha_backup_job.sh | 12 ++++++- role_scripts/13/standby/run.sh | 37 ++++++++++++++++++++-- role_scripts/13/standby/warm_stanby.sh | 13 ++++++-- role_scripts/14/primary/start.sh | 9 ++++-- role_scripts/14/standby/ha_backup_job.sh | 13 ++++++-- role_scripts/14/standby/run.sh | 37 ++++++++++++++++++++-- role_scripts/14/standby/warm_stanby.sh | 13 ++++++-- role_scripts/15/primary/start.sh | 9 ++++-- role_scripts/15/standby/ha_backup_job.sh | 13 ++++++-- role_scripts/15/standby/remote-replica.sh | 13 ++++++-- role_scripts/15/standby/run.sh | 37 ++++++++++++++++++++-- role_scripts/15/standby/warm_stanby.sh | 13 ++++++-- role_scripts/16/primary/start.sh | 9 ++++-- role_scripts/16/standby/ha_backup_job.sh | 13 ++++++-- role_scripts/16/standby/remote-replica.sh | 13 ++++++-- role_scripts/16/standby/run.sh | 38 +++++++++++++++++++++-- role_scripts/16/standby/warm_stanby.sh | 13 ++++++-- role_scripts/9/primary/start.sh | 9 ++++-- role_scripts/9/standby/ha_backup_job.sh | 9 +++++- role_scripts/9/standby/run.sh | 32 +++++++++++++++++-- role_scripts/9/standby/warm_stanby.sh | 8 ++++- scripts/config_recovery.conf.sh | 4 +++ scripts/restore.sh | 7 +++-- 36 files changed, 508 insertions(+), 65 deletions(-) diff --git a/role_scripts/10/primary/start.sh b/role_scripts/10/primary/start.sh index 48d337d..6a1d344 100755 --- a/role_scripts/10/primary/start.sh +++ b/role_scripts/10/primary/start.sh @@ -20,8 +20,13 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf - +#echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf echo "archive_command = '/bin/true'" >>/tmp/postgresql.conf diff --git a/role_scripts/10/standby/ha_backup_job.sh b/role_scripts/10/standby/ha_backup_job.sh index 8b09690..f754665 100755 --- a/role_scripts/10/standby/ha_backup_job.sh +++ b/role_scripts/10/standby/ha_backup_job.sh @@ -77,9 +77,15 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf -echo "shared_preload_libraries = 'pg_stat_statements'" >>/tmp/postgresql.conf +#echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "shared_preload_libraries = 'pg_stat_statements'" >>/tmp/postgresql.conf +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf diff --git a/role_scripts/10/standby/run.sh b/role_scripts/10/standby/run.sh index b24c801..ec71e58 100755 --- a/role_scripts/10/standby/run.sh +++ b/role_scripts/10/standby/run.sh @@ -66,6 +66,30 @@ while true; do sleep 2 done +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + while true; do + echo "Create replication slot on primary" + if [[ "${SSL:-0}" == "ON" ]]; then + output=$(psql -h "$PRIMARY_HOST" --username=postgres "sslmode=$SSL_MODE sslrootcert=/tls/certs/client/ca.crt sslcert=/tls/certs/client/client.crt sslkey=/tls/certs/client/client.key" --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + else + output=$(psql -h "$PRIMARY_HOST" --username=postgres --no-password --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + fi + # check if current pod became leader itself + + if [[ $output == *"(1 row)"* || $output == *"already exists"* ]]; then + break + fi + + if [[ -e "/run_scripts/tmp/pg-failover-trigger" ]]; then + echo "Postgres promotion trigger_file found. Running primary run script" + /run_scripts/role/run.sh + fi + sleep 2 + done +fi + + if [[ ! -e "$PGDATA/PG_VERSION" ]]; then echo "take base basebackup..." # get basebackup @@ -91,14 +115,19 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +#echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf echo "archive_command = '/bin/true'" >>/tmp/postgresql.conf echo "shared_preload_libraries = 'pg_stat_statements'" >>/tmp/postgresql.conf - +echo "max_replication_slots = 90" >>/tmp/postgresql.conf if [ "$STANDBY" == "hot" ]; then echo "hot_standby = on" >>/tmp/postgresql.conf fi diff --git a/role_scripts/10/standby/warm_stanby.sh b/role_scripts/10/standby/warm_stanby.sh index d6fff1f..598ad68 100755 --- a/role_scripts/10/standby/warm_stanby.sh +++ b/role_scripts/10/standby/warm_stanby.sh @@ -22,8 +22,13 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf - +#echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf diff --git a/role_scripts/11/primary/start.sh b/role_scripts/11/primary/start.sh index 1f3c181..dde2bfa 100755 --- a/role_scripts/11/primary/start.sh +++ b/role_scripts/11/primary/start.sh @@ -20,8 +20,13 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf diff --git a/role_scripts/11/standby/ha_backup_job.sh b/role_scripts/11/standby/ha_backup_job.sh index f2b0f71..d337953 100755 --- a/role_scripts/11/standby/ha_backup_job.sh +++ b/role_scripts/11/standby/ha_backup_job.sh @@ -77,8 +77,13 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf diff --git a/role_scripts/11/standby/run.sh b/role_scripts/11/standby/run.sh index 1ae0b69..c2396f5 100755 --- a/role_scripts/11/standby/run.sh +++ b/role_scripts/11/standby/run.sh @@ -66,6 +66,29 @@ while true; do sleep 2 done +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + while true; do + echo "Create replication slot on primary" + if [[ "${SSL:-0}" == "ON" ]]; then + output=$(psql -h "$PRIMARY_HOST" --username=postgres "sslmode=$SSL_MODE sslrootcert=/tls/certs/client/ca.crt sslcert=/tls/certs/client/client.crt sslkey=/tls/certs/client/client.key" --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + else + output=$(psql -h "$PRIMARY_HOST" --username=postgres --no-password --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + fi + # check if current pod became leader itself + + if [[ $output == *"(1 row)"* || $output == *"already exists"* ]]; then + break + fi + + if [[ -e "/run_scripts/tmp/pg-failover-trigger" ]]; then + echo "Postgres promotion trigger_file found. Running primary run script" + /run_scripts/role/run.sh + fi + sleep 2 + done +fi + if [[ ! -e "$PGDATA/PG_VERSION" ]]; then echo "take base basebackup..." # get basebackup @@ -91,8 +114,13 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf diff --git a/role_scripts/11/standby/warm_stanby.sh b/role_scripts/11/standby/warm_stanby.sh index 4c890b9..b29d984 100755 --- a/role_scripts/11/standby/warm_stanby.sh +++ b/role_scripts/11/standby/warm_stanby.sh @@ -22,8 +22,13 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf diff --git a/role_scripts/12/primary/start.sh b/role_scripts/12/primary/start.sh index e91b78e..e85c9c9 100755 --- a/role_scripts/12/primary/start.sh +++ b/role_scripts/12/primary/start.sh @@ -20,8 +20,13 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/12/standby/ha_backup_job.sh b/role_scripts/12/standby/ha_backup_job.sh index ef0fe8c..def35a0 100755 --- a/role_scripts/12/standby/ha_backup_job.sh +++ b/role_scripts/12/standby/ha_backup_job.sh @@ -72,8 +72,18 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +#echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/12/standby/run.sh b/role_scripts/12/standby/run.sh index fa54385..958c671 100755 --- a/role_scripts/12/standby/run.sh +++ b/role_scripts/12/standby/run.sh @@ -66,6 +66,30 @@ while true; do sleep 2 done +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + while true; do + echo "Create replication slot on primary" + if [[ "${SSL:-0}" == "ON" ]]; then + output=$(psql -h "$PRIMARY_HOST" --username=postgres "sslmode=$SSL_MODE sslrootcert=/tls/certs/client/ca.crt sslcert=/tls/certs/client/client.crt sslkey=/tls/certs/client/client.key" --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + else + output=$(psql -h "$PRIMARY_HOST" --username=postgres --no-password --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + fi + # check if current pod became leader itself + + if [[ $output == *"(1 row)"* || $output == *"already exists"* ]]; then + break + fi + + if [[ -e "/run_scripts/tmp/pg-failover-trigger" ]]; then + echo "Postgres promotion trigger_file found. Running primary run script" + /run_scripts/role/run.sh + fi + sleep 2 + done +fi + + if [[ ! -e "$PGDATA/PG_VERSION" ]]; then echo "take base basebackup..." mkdir -p "$PGDATA" @@ -86,7 +110,18 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +# echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi + echo "wal_log_hints = on" >>/tmp/postgresql.conf diff --git a/role_scripts/12/standby/warm_stanby.sh b/role_scripts/12/standby/warm_stanby.sh index 9e48786..56c1f2a 100755 --- a/role_scripts/12/standby/warm_stanby.sh +++ b/role_scripts/12/standby/warm_stanby.sh @@ -20,7 +20,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +# echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf diff --git a/role_scripts/13/primary/start.sh b/role_scripts/13/primary/start.sh index 608825a..07b4307 100755 --- a/role_scripts/13/primary/start.sh +++ b/role_scripts/13/primary/start.sh @@ -20,8 +20,13 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/13/standby/ha_backup_job.sh b/role_scripts/13/standby/ha_backup_job.sh index f7e5e31..88692a1 100755 --- a/role_scripts/13/standby/ha_backup_job.sh +++ b/role_scripts/13/standby/ha_backup_job.sh @@ -72,7 +72,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf diff --git a/role_scripts/13/standby/run.sh b/role_scripts/13/standby/run.sh index ef9a57f..507a8d2 100755 --- a/role_scripts/13/standby/run.sh +++ b/role_scripts/13/standby/run.sh @@ -66,6 +66,30 @@ while true; do sleep 2 done +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + while true; do + echo "Create replication slot on primary" + if [[ "${SSL:-0}" == "ON" ]]; then + output=$(psql -h "$PRIMARY_HOST" --username=postgres "sslmode=$SSL_MODE sslrootcert=/tls/certs/client/ca.crt sslcert=/tls/certs/client/client.crt sslkey=/tls/certs/client/client.key" --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + else + output=$(psql -h "$PRIMARY_HOST" --username=postgres --no-password --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + fi + # check if current pod became leader itself + + if [[ $output == *"(1 row)"* || $output == *"already exists"* ]]; then + break + fi + + if [[ -e "/run_scripts/tmp/pg-failover-trigger" ]]; then + echo "Postgres promotion trigger_file found. Running primary run script" + /run_scripts/role/run.sh + fi + sleep 2 + done +fi + + if [[ ! -e "$PGDATA/PG_VERSION" ]]; then echo "take base basebackup..." mkdir -p "$PGDATA" @@ -86,8 +110,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/13/standby/warm_stanby.sh b/role_scripts/13/standby/warm_stanby.sh index e7a02ca..a06c212 100755 --- a/role_scripts/13/standby/warm_stanby.sh +++ b/role_scripts/13/standby/warm_stanby.sh @@ -20,8 +20,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/14/primary/start.sh b/role_scripts/14/primary/start.sh index 19b7489..8e744bd 100755 --- a/role_scripts/14/primary/start.sh +++ b/role_scripts/14/primary/start.sh @@ -20,8 +20,13 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/14/standby/ha_backup_job.sh b/role_scripts/14/standby/ha_backup_job.sh index 6ca5d3e..ed26ce3 100755 --- a/role_scripts/14/standby/ha_backup_job.sh +++ b/role_scripts/14/standby/ha_backup_job.sh @@ -72,8 +72,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/14/standby/run.sh b/role_scripts/14/standby/run.sh index 2997854..4c4fc99 100755 --- a/role_scripts/14/standby/run.sh +++ b/role_scripts/14/standby/run.sh @@ -66,6 +66,30 @@ while true; do sleep 2 done +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + while true; do + echo "Create replication slot on primary" + if [[ "${SSL:-0}" == "ON" ]]; then + output=$(psql -h "$PRIMARY_HOST" --username=postgres "sslmode=$SSL_MODE sslrootcert=/tls/certs/client/ca.crt sslcert=/tls/certs/client/client.crt sslkey=/tls/certs/client/client.key" --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + else + output=$(psql -h "$PRIMARY_HOST" --username=postgres --no-password --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + fi + # check if current pod became leader itself + + if [[ $output == *"(1 row)"* || $output == *"already exists"* ]]; then + break + fi + + if [[ -e "/run_scripts/tmp/pg-failover-trigger" ]]; then + echo "Postgres promotion trigger_file found. Running primary run script" + /run_scripts/role/run.sh + fi + sleep 2 + done +fi + + if [[ ! -e "$PGDATA/PG_VERSION" ]]; then echo "take base basebackup..." mkdir -p "$PGDATA" @@ -86,8 +110,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/14/standby/warm_stanby.sh b/role_scripts/14/standby/warm_stanby.sh index c1c2161..efa33b1 100755 --- a/role_scripts/14/standby/warm_stanby.sh +++ b/role_scripts/14/standby/warm_stanby.sh @@ -20,8 +20,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name="$CLEAN_HOSTNAME"" >>/tmp/recovery.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/15/primary/start.sh b/role_scripts/15/primary/start.sh index cf6b37e..2e759ae 100755 --- a/role_scripts/15/primary/start.sh +++ b/role_scripts/15/primary/start.sh @@ -20,8 +20,13 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/15/standby/ha_backup_job.sh b/role_scripts/15/standby/ha_backup_job.sh index 49c4aaf..3cd9495 100755 --- a/role_scripts/15/standby/ha_backup_job.sh +++ b/role_scripts/15/standby/ha_backup_job.sh @@ -74,8 +74,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name="$CLEAN_HOSTNAME"" >>/tmp/recovery.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/15/standby/remote-replica.sh b/role_scripts/15/standby/remote-replica.sh index 6540bbc..39c6523 100755 --- a/role_scripts/15/standby/remote-replica.sh +++ b/role_scripts/15/standby/remote-replica.sh @@ -63,8 +63,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 64" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 64" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name="$CLEAN_HOSTNAME"" >>/tmp/recovery.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/15/standby/run.sh b/role_scripts/15/standby/run.sh index e13a0f9..1a3e7d5 100755 --- a/role_scripts/15/standby/run.sh +++ b/role_scripts/15/standby/run.sh @@ -66,6 +66,30 @@ while true; do sleep 2 done +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + while true; do + echo "Create replication slot on primary" + if [[ "${SSL:-0}" == "ON" ]]; then + output=$(psql -h "$PRIMARY_HOST" --username=postgres "sslmode=$SSL_MODE sslrootcert=/tls/certs/client/ca.crt sslcert=/tls/certs/client/client.crt sslkey=/tls/certs/client/client.key" --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + else + output=$(psql -h "$PRIMARY_HOST" --username=postgres --no-password --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + fi + # check if current pod became leader itself + + if [[ $output == *"(1 row)"* || $output == *"already exists"* ]]; then + break + fi + + if [[ -e "/run_scripts/tmp/pg-failover-trigger" ]]; then + echo "Postgres promotion trigger_file found. Running primary run script" + /run_scripts/role/run.sh + fi + sleep 2 + done +fi + + if [[ ! -e "$PGDATA/PG_VERSION" ]]; then echo "take base basebackup..." mkdir -p "$PGDATA" @@ -86,8 +110,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/15/standby/warm_stanby.sh b/role_scripts/15/standby/warm_stanby.sh index 125f253..ed0c6a1 100755 --- a/role_scripts/15/standby/warm_stanby.sh +++ b/role_scripts/15/standby/warm_stanby.sh @@ -20,8 +20,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/16/primary/start.sh b/role_scripts/16/primary/start.sh index b09af3c..648cc8f 100755 --- a/role_scripts/16/primary/start.sh +++ b/role_scripts/16/primary/start.sh @@ -20,10 +20,15 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi echo "wal_log_hints = on" >>/tmp/postgresql.conf - +echo "max_replication_slots = 90" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. echo "archive_mode = always" >>/tmp/postgresql.conf if [[ "${WAL_BACKUP_TYPE:-0}" == "WALG" ]]; then diff --git a/role_scripts/16/standby/ha_backup_job.sh b/role_scripts/16/standby/ha_backup_job.sh index dc7cfd1..05abab8 100755 --- a/role_scripts/16/standby/ha_backup_job.sh +++ b/role_scripts/16/standby/ha_backup_job.sh @@ -74,8 +74,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/16/standby/remote-replica.sh b/role_scripts/16/standby/remote-replica.sh index 7c95aa4..04ad8ec 100755 --- a/role_scripts/16/standby/remote-replica.sh +++ b/role_scripts/16/standby/remote-replica.sh @@ -62,8 +62,17 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS - -echo "wal_keep_size = 64" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +echo "max_replication_slots = 90" >>/tmp/postgresql.conf +# echo "wal_keep_size = 64" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi echo "wal_log_hints = on" >>/tmp/postgresql.conf diff --git a/role_scripts/16/standby/run.sh b/role_scripts/16/standby/run.sh index 29dd11d..cc96d72 100755 --- a/role_scripts/16/standby/run.sh +++ b/role_scripts/16/standby/run.sh @@ -66,6 +66,29 @@ while true; do sleep 2 done +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + while true; do + echo "Create replication slot on primary" + if [[ "${SSL:-0}" == "ON" ]]; then + output=$(psql -h "$PRIMARY_HOST" --username=postgres "sslmode=$SSL_MODE sslrootcert=/tls/certs/client/ca.crt sslcert=/tls/certs/client/client.crt sslkey=/tls/certs/client/client.key" --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + else + output=$(psql -h "$PRIMARY_HOST" --username=postgres --no-password --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + fi + # check if current pod became leader itself + + if [[ $output == *"(1 row)"* || $output == *"already exists"* ]]; then + break + fi + + if [[ -e "/run_scripts/tmp/pg-failover-trigger" ]]; then + echo "Postgres promotion trigger_file found. Running primary run script" + /run_scripts/role/run.sh + fi + sleep 2 + done +fi + if [[ ! -e "$PGDATA/PG_VERSION" ]]; then echo "take base basebackup..." mkdir -p "$PGDATA" @@ -80,14 +103,25 @@ else /run_scripts/role/warm_stanby.sh fi + + # setup postgresql.conf touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 - +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. diff --git a/role_scripts/16/standby/warm_stanby.sh b/role_scripts/16/standby/warm_stanby.sh index 98c0ab7..538c854 100755 --- a/role_scripts/16/standby/warm_stanby.sh +++ b/role_scripts/16/standby/warm_stanby.sh @@ -20,10 +20,19 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +# echo "wal_keep_size = 1024" >>/tmp/postgresql.conf #it was "wal_keep_segments" in earlier version. changed in version 13 +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf +fi +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/postgresql.conf +fi echo "wal_log_hints = on" >>/tmp/postgresql.conf - +echo "max_replication_slots = 90" >>/tmp/postgresql.conf # we are not doing any archiving by default but it's better to have this config in our postgresql.conf file in case of customization. echo "archive_mode = always" >>/tmp/postgresql.conf echo "archive_command = '/bin/true'" >>/tmp/postgresql.conf diff --git a/role_scripts/9/primary/start.sh b/role_scripts/9/primary/start.sh index e3197c0..63ecbec 100755 --- a/role_scripts/9/primary/start.sh +++ b/role_scripts/9/primary/start.sh @@ -20,12 +20,17 @@ echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf echo "archive_command = '/bin/true'" >>/tmp/postgresql.conf echo "shared_preload_libraries = 'pg_stat_statements'" >>/tmp/postgresql.conf +echo "max_replication_slots = 90" >>/tmp/postgresql.conf if [[ "${SSL:-0}" == "ON" ]]; then echo "ssl = on" >>/tmp/postgresql.conf diff --git a/role_scripts/9/standby/ha_backup_job.sh b/role_scripts/9/standby/ha_backup_job.sh index c8c6530..b94ae61 100755 --- a/role_scripts/9/standby/ha_backup_job.sh +++ b/role_scripts/9/standby/ha_backup_job.sh @@ -77,10 +77,17 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +#echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi + echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "shared_preload_libraries = 'pg_stat_statements'" >>/tmp/postgresql.conf +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf echo "archive_command = '/bin/true'" >>/tmp/postgresql.conf diff --git a/role_scripts/9/standby/run.sh b/role_scripts/9/standby/run.sh index effaf7c..74788c5 100755 --- a/role_scripts/9/standby/run.sh +++ b/role_scripts/9/standby/run.sh @@ -66,6 +66,29 @@ while true; do sleep 2 done +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + while true; do + echo "Create replication slot on primary" + if [[ "${SSL:-0}" == "ON" ]]; then + output=$(psql -h "$PRIMARY_HOST" --username=postgres "sslmode=$SSL_MODE sslrootcert=/tls/certs/client/ca.crt sslcert=/tls/certs/client/client.crt sslkey=/tls/certs/client/client.key" --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + else + output=$(psql -h "$PRIMARY_HOST" --username=postgres --no-password --command="SELECT pg_create_physical_replication_slot('${CLEAN_HOSTNAME}', true);" 2>&1 || true) + fi + # check if current pod became leader itself + + if [[ $output == *"(1 row)"* || $output == *"already exists"* ]]; then + break + fi + + if [[ -e "/run_scripts/tmp/pg-failover-trigger" ]]; then + echo "Postgres promotion trigger_file found. Running primary run script" + /run_scripts/role/run.sh + fi + sleep 2 + done +fi + if [[ ! -e "$PGDATA/PG_VERSION" ]]; then echo "take base basebackup..." # get basebackup @@ -91,8 +114,13 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf - +# echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "archive_mode = always" >>/tmp/postgresql.conf diff --git a/role_scripts/9/standby/warm_stanby.sh b/role_scripts/9/standby/warm_stanby.sh index b4387da..73a8305 100755 --- a/role_scripts/9/standby/warm_stanby.sh +++ b/role_scripts/9/standby/warm_stanby.sh @@ -22,7 +22,13 @@ touch /tmp/postgresql.conf echo "wal_level = replica" >>/tmp/postgresql.conf echo "shared_buffers = $SHARED_BUFFERS" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS -echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +#echo "wal_keep_segments = 1024" >>/tmp/postgresql.conf +if [ ! -z "${WAL_RETAIN_PARAM:-}" ] && [ ! -z "${WAL_RETAIN_AMOUNT:-}" ]; then + echo "${WAL_RETAIN_PARAM}=${WAL_RETAIN_AMOUNT}" >>/tmp/postgresql.conf +else + echo "wal_keep_segments = 64" >>/tmp/postgresql.conf +fi +echo "max_replication_slots = 90" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf echo "shared_preload_libraries = 'pg_stat_statements'" >>/tmp/postgresql.conf diff --git a/scripts/config_recovery.conf.sh b/scripts/config_recovery.conf.sh index d6a187f..a4978fc 100755 --- a/scripts/config_recovery.conf.sh +++ b/scripts/config_recovery.conf.sh @@ -6,6 +6,10 @@ echo "standby_mode = on" >>/tmp/recovery.conf echo "trigger_file = '/run_scripts/tmp/pg-failover-trigger'" >>/tmp/recovery.conf echo "recovery_target_timeline = 'latest'" >>/tmp/recovery.conf +if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then + CLEAN_HOSTNAME="${HOSTNAME//[^[:alnum:]]/}" + echo "primary_slot_name = "$CLEAN_HOSTNAME"" >>/tmp/recovery.conf +fi # primary_conninfo is used for streaming replication if [[ "${SSL:-0}" == "ON" ]]; then if [[ "$CLIENT_AUTH_MODE" == "cert" ]]; then diff --git a/scripts/restore.sh b/scripts/restore.sh index 77437f6..5af8f5c 100755 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -33,8 +33,8 @@ if [[ "$PG_MAJOR" == "11" ]]; then echo "wal_level = replica" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS + # echo "wal_keep_segments = 64" >>/tmp/postgresql.conf echo "wal_keep_segments = 64" >>/tmp/postgresql.conf - echo "wal_log_hints = on" >>/tmp/postgresql.conf else # ****************** Recovery config 12, 13, 14 ************************** @@ -53,7 +53,8 @@ else echo "wal_level = replica" >>/tmp/postgresql.conf echo "max_wal_senders = 90" >>/tmp/postgresql.conf # default is 10. value must be less than max_connections minus superuser_reserved_connections. ref: https://www.postgresql.org/docs/11/runtime-config-replication.html#GUC-MAX-WAL-SENDERS - echo "wal_keep_size = 64" >>/tmp/postgresql.conf + # echo "wal_keep_size = 64" >>/tmp/postgresql.conf + echo "wal_keep_size = 1024" >>/tmp/postgresql.conf echo "hot_standby = on" >>/tmp/postgresql.conf echo "wal_log_hints = on" >>/tmp/postgresql.conf @@ -66,7 +67,7 @@ echo "archive_command = '/bin/true'" >>/tmp/postgresql.conf cat /run_scripts/role/postgresql.conf >>/tmp/postgresql.conf mv /tmp/postgresql.conf "$PGDATA/postgresql.conf" - +echo "max_replication_slots = 90" >>/tmp/postgresql.conf # setup pg_hba.conf for initial start. this one is just for initialization touch /tmp/pg_hba.conf { echo '#TYPE DATABASE USER ADDRESS METHOD'; } >>tmp/pg_hba.conf