add the ability to use a secret persistence (airbytehq#6415)

* test exposing secrets in configrepo

* fix local persistence sql

* working propagation, just without check/discover replacements and without feature flagging

* switch if statement

* set up secret persistence for google secrets manager

* add ttl-based secret persistence for check/discover usage in the future

* set up check/discover to pass around necessary parts

* Revert "set up check/discover to pass around necessary parts"

This reverts commit 489d2d5.

* working updates + check/discover operations

* fix additional configs created on deletion

* clean up docker compose file

* finish up configrepo

* make api path optional

* clean up schedulerapp and local testing persistence

* make optional in the worker app

* add rest of feature flagging

* fmt

* remove completed todo

* fix refactoring typo

* fix another refactoring typo

* fix compilation error in test case

* fix tests

* final cleanups

* fix conditional

* address a couple of things

* add hydrator interface

* add replaceAllConfigs

* specfetcher handling

* fix constructor

* fix test

* fix typo

* fix merge build error

* remove extra config

* fix integration test

* fix final piece

.env

@@ -75,3 +75,6 @@ MAX_SYNC_JOB_ATTEMPTS=3
 
 # Time in days to reach a timeout to cancel the synchronization
 MAX_SYNC_TIMEOUT_DAYS=3
+
+# Set secret persistence store to use. Do not change this for existing installations!
+SECRET_PERSISTENCE=NONE

airbyte-config/models/src/main/java/io/airbyte/config/Configs.java

@@ -43,8 +43,6 @@ public interface Configs {
 
   String getSecretStoreGcpCredentials();
 
-  String getSecretStoreForConfigs();
-
   boolean runDatabaseMigrationOnStartup();
 
   int getMaxSyncJobAttempts();
@@ -109,6 +107,8 @@ public interface Configs {
 
   String getGoogleApplicationCredentials();
 
+  SecretPersistenceType getSecretPersistenceType();
+
   enum TrackingStrategy {
     SEGMENT,
     LOGGING
@@ -124,4 +124,10 @@ enum DeploymentMode {
     CLOUD
   }
 
+  enum SecretPersistenceType {
+    NONE,
+    TESTING_CONFIG_DB_TABLE,
+    GOOGLE_SECRET_MANAGER
+  }
+
 }

airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java

@@ -67,6 +67,7 @@ public class EnvConfigs implements Configs {
   private static final String RESOURCE_CPU_LIMIT = "RESOURCE_CPU_LIMIT";
   private static final String RESOURCE_MEMORY_REQUEST = "RESOURCE_MEMORY_REQUEST";
   private static final String RESOURCE_MEMORY_LIMIT = "RESOURCE_MEMORY_LIMIT";
+  private static final String SECRET_PERSISTENCE = "SECRET_PERSISTENCE";
   private static final String JOBS_IMAGE_PULL_SECRET = "JOBS_IMAGE_PULL_SECRET";
 
   // defaults
@@ -76,7 +77,6 @@ public class EnvConfigs implements Configs {
   private static final String DEFAULT_RESOURCE_REQUIREMENT_MEMORY = null;
   private static final String SECRET_STORE_GCP_PROJECT_ID = "SECRET_STORE_GCP_PROJECT_ID";
   private static final String SECRET_STORE_GCP_CREDENTIALS = "SECRET_STORE_GCP_CREDENTIALS";
-  private static final String SECRET_STORE_FOR_CONFIGS = "SECRET_STORE_CONFIGS_ENABLE";
   private static final long DEFAULT_MINIMUM_WORKSPACE_RETENTION_DAYS = 1;
   private static final long DEFAULT_MAXIMUM_WORKSPACE_RETENTION_DAYS = 60;
   private static final long DEFAULT_MAXIMUM_WORKSPACE_SIZE_MB = 5000;
@@ -191,11 +191,6 @@ public String getSecretStoreGcpProjectId() {
     return getEnv(SECRET_STORE_GCP_PROJECT_ID);
   }
 
-  @Override
-  public String getSecretStoreForConfigs() {
-    return getEnv(SECRET_STORE_FOR_CONFIGS);
-  }
-
   @Override
   public boolean runDatabaseMigrationOnStartup() {
     return getEnvOrDefault(RUN_DATABASE_MIGRATION_ON_STARTUP, true);
@@ -428,6 +423,12 @@ public String getGoogleApplicationCredentials() {
     return getEnvOrDefault(LogClientSingleton.GOOGLE_APPLICATION_CREDENTIALS, "");
   }
 
+  @Override
+  public SecretPersistenceType getSecretPersistenceType() {
+    final var secretPersistenceStr = getEnvOrDefault(SECRET_PERSISTENCE, SecretPersistenceType.NONE.name());
+    return SecretPersistenceType.valueOf(secretPersistenceStr);
+  }
+
   private String getEnvOrDefault(final String key, final String defaultValue) {
     return getEnvOrDefault(key, defaultValue, Function.identity(), false);
   }

airbyte-config/persistence/build.gradle

@@ -6,6 +6,7 @@ plugins {
 dependencies {
     implementation 'commons-io:commons-io:2.7'
 
+    implementation project(':airbyte-commons-docker')
     implementation project(':airbyte-db:lib')
     implementation project(':airbyte-db:jooq')
     implementation project(':airbyte-protocol:models')