Make spec field required on definitions + remove SpecFetcher usage (a…

…irbytehq#8140)

* make spec field required

* remove spec backfill logic

* remove usages of specFetcher.getSpec()

* remove unused code and the caching scheduler clients

* fix tests to work with fetching specs from definitions

* fetch spec from definition in config repository and fix RunMigrationTest

* remove unused SpecFetcher methods/tests

* run gw format

* run gw format

* undo change to main method

* add back newlines

* set additional properties to true on destination definition

* remove now-unused VERSION_0_32_0_FORCE_UPGRADE env var

airbyte-config/models/src/main/java/io/airbyte/config/Configs.java

@@ -121,8 +121,6 @@ public interface Configs {
 
   boolean getPublishMetrics();
 
-  boolean getVersion32ForceUpgrade();
-
   SecretPersistenceType getSecretPersistenceType();
 
   enum TrackingStrategy {

airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java

@@ -77,7 +77,6 @@ public class EnvConfigs implements Configs {
   private static final String SECRET_PERSISTENCE = "SECRET_PERSISTENCE";
   private static final String JOBS_IMAGE_PULL_SECRET = "JOBS_IMAGE_PULL_SECRET";
   private static final String PUBLISH_METRICS = "PUBLISH_METRICS";
-  private static final String VERSION_0_32_0_FORCE_UPGRADE = "VERSION_0_32_0_FORCE_UPGRADE";
 
   // defaults
   private static final String DEFAULT_SPEC_CACHE_BUCKET = "io-airbyte-cloud-spec-cache";
@@ -478,11 +477,6 @@ public boolean getPublishMetrics() {
     return getEnvOrDefault(PUBLISH_METRICS, false);
   }
 
-  @Override
-  public boolean getVersion32ForceUpgrade() {
-    return getEnvOrDefault(VERSION_0_32_0_FORCE_UPGRADE, false);
-  }
-
   @Override
   public SecretPersistenceType getSecretPersistenceType() {
     final var secretPersistenceStr = getEnvOrDefault(SECRET_PERSISTENCE, SecretPersistenceType.NONE.name());

airbyte-config/models/src/main/resources/types/StandardDestinationDefinition.yaml

@@ -10,7 +10,8 @@ required:
   - dockerRepository
   - dockerImageTag
   - documentationUrl
-additionalProperties: false
+  - spec
+additionalProperties: true
 properties:
   destinationDefinitionId:
     type: string

airbyte-config/models/src/main/resources/types/StandardSourceDefinition.yaml

@@ -10,6 +10,7 @@ required:
   - dockerRepository
   - dockerImageTag
   - documentationUrl
+  - spec
 additionalProperties: true
 properties:
   sourceDefinitionId:

airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java

@@ -5,8 +5,6 @@
 package io.airbyte.config.persistence;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.google.api.client.util.Preconditions;
-import io.airbyte.commons.docker.DockerUtils;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.lang.Exceptions;
 import io.airbyte.commons.lang.MoreBooleans;
@@ -57,7 +55,6 @@ public class ConfigRepository {
   private final SecretsHydrator secretsHydrator;
   private final Optional<SecretPersistence> longLivedSecretPersistence;
   private final Optional<SecretPersistence> ephemeralSecretPersistence;
-  private Function<String, ConnectorSpecification> specFetcherFn;
 
   public ConfigRepository(final ConfigPersistence persistence,
                           final SecretsHydrator secretsHydrator,
@@ -547,7 +544,6 @@ public void replaceAllConfigsDeserializing(final Map<String, Stream<JsonNode>> c
 
   public void replaceAllConfigs(final Map<AirbyteConfig, Stream<?>> configs, final boolean dryRun) throws IOException {
     if (longLivedSecretPersistence.isPresent()) {
-      Preconditions.checkNotNull(specFetcherFn);
       final var augmentedMap = new HashMap<>(configs);
 
       // get all source defs so that we can use their specs when storing secrets.
@@ -558,11 +554,7 @@ public void replaceAllConfigs(final Map<AirbyteConfig, Stream<?>> configs, final
       augmentedMap.put(ConfigSchema.STANDARD_SOURCE_DEFINITION, sourceDefs.stream());
       final Map<UUID, ConnectorSpecification> sourceDefIdToSpec = sourceDefs
           .stream()
-          .collect(Collectors.toMap(StandardSourceDefinition::getSourceDefinitionId, sourceDefinition -> {
-            final String imageName = DockerUtils
-                .getTaggedImageName(sourceDefinition.getDockerRepository(), sourceDefinition.getDockerImageTag());
-            return specFetcherFn.apply(imageName);
-          }));
+          .collect(Collectors.toMap(StandardSourceDefinition::getSourceDefinitionId, StandardSourceDefinition::getSpec));
 
       // get all destination defs so that we can use their specs when storing secrets.
       @SuppressWarnings("unchecked")
@@ -571,11 +563,7 @@ public void replaceAllConfigs(final Map<AirbyteConfig, Stream<?>> configs, final
       augmentedMap.put(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destinationDefs.stream());
       final Map<UUID, ConnectorSpecification> destinationDefIdToSpec = destinationDefs
           .stream()
-          .collect(Collectors.toMap(StandardDestinationDefinition::getDestinationDefinitionId, destinationDefinition -> {
-            final String imageName = DockerUtils
-                .getTaggedImageName(destinationDefinition.getDockerRepository(), destinationDefinition.getDockerImageTag());
-            return specFetcherFn.apply(imageName);
-          }));
+          .collect(Collectors.toMap(StandardDestinationDefinition::getDestinationDefinitionId, StandardDestinationDefinition::getSpec));
 
       if (augmentedMap.containsKey(ConfigSchema.SOURCE_CONNECTION)) {
         final Stream<?> augmentedValue = augmentedMap.get(ConfigSchema.SOURCE_CONNECTION)
@@ -640,8 +628,4 @@ public void loadData(final ConfigPersistence seedPersistence) throws IOException
     persistence.loadData(seedPersistence);
   }
 
-  public void setSpecFetcher(final Function<String, ConnectorSpecification> specFetcherFn) {
-    this.specFetcherFn = specFetcherFn;
-  }
-
 }

airbyte-migration/src/main/resources/migrations/migrationV0_14_0/airbyte_config/StandardDestinationDefinition.yaml

@@ -10,7 +10,7 @@ required:
   - dockerRepository
   - dockerImageTag
   - documentationUrl
-additionalProperties: false
+additionalProperties: true
 properties:
   destinationDefinitionId:
     type: string

airbyte-migration/src/main/resources/migrations/migrationV0_20_0/StandardDestinationDefinition.yaml

@@ -10,7 +10,7 @@ required:
   - dockerRepository
   - dockerImageTag
   - documentationUrl
-additionalProperties: false
+additionalProperties: true
 properties:
   destinationDefinitionId:
     type: string