From b1e6d5a083a2bbb3d5e45e9c2ae05ed0b75fd7bf Mon Sep 17 00:00:00 2001 From: Joshua Jerin Date: Mon, 25 Nov 2024 18:46:30 +0300 Subject: [PATCH 1/6] initial commit --- tembo-operator/src/cloudnativepg/hibernate.rs | 17 +++++++++++++++++ tembo-operator/src/ingress.rs | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tembo-operator/src/cloudnativepg/hibernate.rs b/tembo-operator/src/cloudnativepg/hibernate.rs index c49df7f43..0b5d351a9 100644 --- a/tembo-operator/src/cloudnativepg/hibernate.rs +++ b/tembo-operator/src/cloudnativepg/hibernate.rs @@ -3,6 +3,7 @@ use crate::cloudnativepg::clusters::{ClusterStatusConditions, ClusterStatusCondi use crate::cloudnativepg::cnpg::{get_cluster, get_pooler, get_scheduled_backups}; use crate::cloudnativepg::poolers::Pooler; use crate::cloudnativepg::scheduledbackups::ScheduledBackup; +use crate::ingress::delete_ingress_route_tcp; use crate::Error; use crate::{patch_cdb_status_merge, requeue_normal_with_jitter, Context}; @@ -145,6 +146,22 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> } } + // Delete the IngressRouteTCP route for hibernated instances + if let Err(err) = delete_ingress_route_tcp( + Api::namespaced(client, &namespace), + &namespace, + &format!("{}-ro-0", cdb.name_any().as_str()), + ) + .await + { + warn!( + "Error deleting ingress route for {}: {}", + cdb.name_any(), + err + ); + return Err(Action::requeue(Duration::from_secs(300))); + } + // Stop CNPG reconciliation for hibernated instances. // We should not stop CNPG reconciliation until hibernation is fully completed, // as the instance may not finish hibernating otherwise. diff --git a/tembo-operator/src/ingress.rs b/tembo-operator/src/ingress.rs index 1d7f67128..e1da9a22d 100644 --- a/tembo-operator/src/ingress.rs +++ b/tembo-operator/src/ingress.rs @@ -196,7 +196,7 @@ async fn apply_ingress_route_tcp( Ok(()) } -async fn delete_ingress_route_tcp( +pub async fn delete_ingress_route_tcp( ingress_route_tcp_api: Api, namespace: &str, ingress_route_tcp_name: &String, From b9bbc7d0855bf53465697680a1b3115e18aff781 Mon Sep 17 00:00:00 2001 From: Joshua Jerin Date: Mon, 25 Nov 2024 18:54:58 +0300 Subject: [PATCH 2/6] fmt change --- tembo-operator/src/cloudnativepg/hibernate.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tembo-operator/src/cloudnativepg/hibernate.rs b/tembo-operator/src/cloudnativepg/hibernate.rs index 0b5d351a9..5b4e5979a 100644 --- a/tembo-operator/src/cloudnativepg/hibernate.rs +++ b/tembo-operator/src/cloudnativepg/hibernate.rs @@ -146,11 +146,12 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> } } - // Delete the IngressRouteTCP route for hibernated instances + // Remove IngressRouteTCP route for stopped instances + let ingress_route_tcp_name = format!("{}-ro-0", cdb.name_any().as_str()); if let Err(err) = delete_ingress_route_tcp( Api::namespaced(client, &namespace), &namespace, - &format!("{}-ro-0", cdb.name_any().as_str()), + &ingress_route_tcp_name, ) .await { From 83bd9d3119ed7129631b4a035df07ffa4ab76a53 Mon Sep 17 00:00:00 2001 From: Joshua Jerin Date: Mon, 25 Nov 2024 19:26:57 +0300 Subject: [PATCH 3/6] requested changes --- tembo-operator/src/cloudnativepg/hibernate.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tembo-operator/src/cloudnativepg/hibernate.rs b/tembo-operator/src/cloudnativepg/hibernate.rs index 5b4e5979a..563c870e8 100644 --- a/tembo-operator/src/cloudnativepg/hibernate.rs +++ b/tembo-operator/src/cloudnativepg/hibernate.rs @@ -148,12 +148,10 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> // Remove IngressRouteTCP route for stopped instances let ingress_route_tcp_name = format!("{}-ro-0", cdb.name_any().as_str()); - if let Err(err) = delete_ingress_route_tcp( - Api::namespaced(client, &namespace), - &namespace, - &ingress_route_tcp_name, - ) - .await + let ingress_route_tcp_api = Api::namespaced(ctx.client.clone(), &namespace); + + if let Err(err) = + delete_ingress_route_tcp(ingress_route_tcp_api, &namespace, &ingress_route_tcp_name).await { warn!( "Error deleting ingress route for {}: {}", From c55d796502fc6e20c98d76f0214d40e52a2b5c35 Mon Sep 17 00:00:00 2001 From: Joshua Jerin Date: Tue, 26 Nov 2024 17:13:45 +0300 Subject: [PATCH 4/6] requested changes 2 --- tembo-operator/src/cloudnativepg/hibernate.rs | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tembo-operator/src/cloudnativepg/hibernate.rs b/tembo-operator/src/cloudnativepg/hibernate.rs index 563c870e8..9524e542d 100644 --- a/tembo-operator/src/cloudnativepg/hibernate.rs +++ b/tembo-operator/src/cloudnativepg/hibernate.rs @@ -147,11 +147,10 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> } // Remove IngressRouteTCP route for stopped instances - let ingress_route_tcp_name = format!("{}-ro-0", cdb.name_any().as_str()); let ingress_route_tcp_api = Api::namespaced(ctx.client.clone(), &namespace); - + let prefix_read_only = format!("{}-ro-0", cdb.name_any().as_str()); if let Err(err) = - delete_ingress_route_tcp(ingress_route_tcp_api, &namespace, &ingress_route_tcp_name).await + delete_ingress_route_tcp(ingress_route_tcp_api.clone(), &namespace, &prefix_read_only).await { warn!( "Error deleting ingress route for {}: {}", @@ -161,6 +160,34 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> return Err(Action::requeue(Duration::from_secs(300))); } + let prefix_read_write = format!("{}-rw-0", cdb.name_any().as_str()); + if let Err(err) = delete_ingress_route_tcp( + ingress_route_tcp_api.clone(), + &namespace, + &prefix_read_write, + ) + .await + { + warn!( + "Error deleting extra ingress route for {}: {}", + cdb.name_any(), + err + ); + return Err(Action::requeue(Duration::from_secs(300))); + } + + let prefix_pooler = format!("{}-poller-0", cdb.name_any().as_str()); + if let Err(err) = + delete_ingress_route_tcp(ingress_route_tcp_api.clone(), &namespace, &prefix_pooler).await + { + warn!( + "Error deleting pooler ingress route for {}: {}", + cdb.name_any(), + err + ); + return Err(Action::requeue(Duration::from_secs(300))); + } + // Stop CNPG reconciliation for hibernated instances. // We should not stop CNPG reconciliation until hibernation is fully completed, // as the instance may not finish hibernating otherwise. From 691b30efa8dd39392f6156703ebafbb40e5715db Mon Sep 17 00:00:00 2001 From: Joshua Jerin Date: Tue, 26 Nov 2024 18:04:07 +0300 Subject: [PATCH 5/6] cover custom domain --- tembo-operator/src/cloudnativepg/hibernate.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tembo-operator/src/cloudnativepg/hibernate.rs b/tembo-operator/src/cloudnativepg/hibernate.rs index 9524e542d..523f9b3c8 100644 --- a/tembo-operator/src/cloudnativepg/hibernate.rs +++ b/tembo-operator/src/cloudnativepg/hibernate.rs @@ -153,7 +153,7 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> delete_ingress_route_tcp(ingress_route_tcp_api.clone(), &namespace, &prefix_read_only).await { warn!( - "Error deleting ingress route for {}: {}", + "Error deleting postgres ingress route for {}: {}", cdb.name_any(), err ); @@ -169,7 +169,7 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> .await { warn!( - "Error deleting extra ingress route for {}: {}", + "Error deleting postgres ingress route for {}: {}", cdb.name_any(), err ); @@ -181,7 +181,19 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> delete_ingress_route_tcp(ingress_route_tcp_api.clone(), &namespace, &prefix_pooler).await { warn!( - "Error deleting pooler ingress route for {}: {}", + "Error deleting postgres ingress route for {}: {}", + cdb.name_any(), + err + ); + return Err(Action::requeue(Duration::from_secs(300))); + } + + let prefix_extra = format!("extra-{}-rw", cdb.name_any().as_str()); + if let Err(err) = + delete_ingress_route_tcp(ingress_route_tcp_api.clone(), &namespace, &prefix_extra).await + { + warn!( + "Error deleting extra postgres ingress route for {}: {}", cdb.name_any(), err ); From 60ae97bc9cbe58c7de1fc484ab07d4c668723638 Mon Sep 17 00:00:00 2001 From: Joshua Jerin Date: Wed, 27 Nov 2024 00:25:57 +0300 Subject: [PATCH 6/6] requested changes 3 --- tembo-operator/src/cloudnativepg/hibernate.rs | 25 +++++++++++-------- tembo-operator/tests/integration_tests.rs | 14 +++++++++++ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/tembo-operator/src/cloudnativepg/hibernate.rs b/tembo-operator/src/cloudnativepg/hibernate.rs index 523f9b3c8..340ed019e 100644 --- a/tembo-operator/src/cloudnativepg/hibernate.rs +++ b/tembo-operator/src/cloudnativepg/hibernate.rs @@ -176,7 +176,7 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> return Err(Action::requeue(Duration::from_secs(300))); } - let prefix_pooler = format!("{}-poller-0", cdb.name_any().as_str()); + let prefix_pooler = format!("{}-pooler-0", cdb.name_any().as_str()); if let Err(err) = delete_ingress_route_tcp(ingress_route_tcp_api.clone(), &namespace, &prefix_pooler).await { @@ -188,16 +188,19 @@ pub async fn reconcile_cluster_hibernation(cdb: &CoreDB, ctx: &Arc) -> return Err(Action::requeue(Duration::from_secs(300))); } - let prefix_extra = format!("extra-{}-rw", cdb.name_any().as_str()); - if let Err(err) = - delete_ingress_route_tcp(ingress_route_tcp_api.clone(), &namespace, &prefix_extra).await - { - warn!( - "Error deleting extra postgres ingress route for {}: {}", - cdb.name_any(), - err - ); - return Err(Action::requeue(Duration::from_secs(300))); + let extra_domain_names = cdb.spec.extra_domains_rw.clone().unwrap_or_default(); + if !extra_domain_names.is_empty() { + let prefix_extra = format!("extra-{}-rw", cdb.name_any().as_str()); + if let Err(err) = + delete_ingress_route_tcp(ingress_route_tcp_api.clone(), &namespace, &prefix_extra).await + { + warn!( + "Error deleting extra postgres ingress route for {}: {}", + cdb.name_any(), + err + ); + return Err(Action::requeue(Duration::from_secs(300))); + } } // Stop CNPG reconciliation for hibernated instances. diff --git a/tembo-operator/tests/integration_tests.rs b/tembo-operator/tests/integration_tests.rs index 53f00a9d0..d570ca3bf 100644 --- a/tembo-operator/tests/integration_tests.rs +++ b/tembo-operator/tests/integration_tests.rs @@ -5808,6 +5808,7 @@ CREATE EVENT TRIGGER pgrst_watch let test = TestCore::new(test_name).await; let name = test.name.clone(); let pooler_name = format!("{}-pooler", name); + let namespace = test.namespace.clone(); // Generate very simple CoreDB JSON definitions. The first will be for // initializing and starting the cluster, and the second for stopping @@ -5850,6 +5851,19 @@ CREATE EVENT TRIGGER pgrst_watch .await .not()); + // Assert that ingress routes are removed after hibernation + + let client = test.client.clone(); + let ingresses_tcp: Vec = + list_resources(client.clone(), &name, &namespace, 0) + .await + .unwrap(); + assert_eq!( + ingresses_tcp.len(), + 0, + "Ingress routes should be removed after hibernation" + ); + // Patch the cluster to start it up again, then check to ensure it // actually did so. This proves hibernation can be reversed.