diff --git a/.gitignore b/.gitignore index 95c4c345acd..1f37aca5424 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ secrets updated_configurations !airbyte-integrations/connector-templates/**/secrets +# include airbyte-data +!airbyte-data/src/main/java/io/airbyte/data + # Test logs acceptance_tests_logs diff --git a/airbyte-analytics/build.gradle b/airbyte-analytics/build.gradle deleted file mode 100644 index 4eb74238c53..00000000000 --- a/airbyte-analytics/build.gradle +++ /dev/null @@ -1,23 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - api libs.segment.java.analytics - api libs.micronaut.http - - implementation libs.bundles.jackson - implementation libs.guava - - implementation project(':airbyte-commons') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-data') - implementation project(':airbyte-json-validation') - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - testImplementation libs.junit.pioneer -} diff --git a/airbyte-analytics/build.gradle.kts b/airbyte-analytics/build.gradle.kts new file mode 100644 index 00000000000..7fef3aee1e1 --- /dev/null +++ b/airbyte-analytics/build.gradle.kts @@ -0,0 +1,23 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + api(libs.segment.java.analytics) + api(libs.micronaut.http) + + implementation(libs.bundles.jackson) + implementation(libs.guava) + + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-data")) + implementation(project(":airbyte-json-validation")) + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + testImplementation(libs.junit.pioneer) +} diff --git a/airbyte-api-server/build.gradle b/airbyte-api-server/build.gradle deleted file mode 100644 index 9cead40234f..00000000000 --- a/airbyte-api-server/build.gradle +++ /dev/null @@ -1,86 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.app" - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - id "org.jetbrains.kotlin.jvm" - id "org.jetbrains.kotlin.kapt" - id "io.airbyte.gradle.docker" -} - -dependencies { - implementation 'org.apache.logging.log4j:log4j-slf4j2-impl' - - kapt(platform(libs.micronaut.bom)) - kapt(libs.bundles.micronaut.annotation.processor) - - kaptTest(platform(libs.micronaut.bom)) - kaptTest(libs.bundles.micronaut.test.annotation.processor) - - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - annotationProcessor libs.micronaut.jaxrs.processor - - implementation project(':airbyte-analytics') - implementation project(':airbyte-api') - implementation project(':airbyte-commons') - implementation project(':airbyte-config:config-models') - implementation 'com.cronutils:cron-utils:9.2.1' - implementation libs.bundles.jackson - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.bundles.micronaut.data.jdbc - implementation libs.micronaut.jaxrs.server - implementation libs.micronaut.problem.json - implementation libs.micronaut.security - - implementation libs.sentry.java - implementation libs.swagger.annotations - implementation libs.javax.ws.rs.api - - runtimeOnly libs.javax.databind - - testImplementation libs.bundles.micronaut.test - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - testImplementation project(':airbyte-test-utils') - testImplementation libs.bundles.micronaut.test - testImplementation libs.postgresql - testImplementation libs.platform.testcontainers.postgresql - testImplementation libs.mockwebserver - testImplementation libs.mockito.inline - - implementation libs.airbyte.protocol - -} - -kapt { - correctErrorTypes true -} - -Properties env = new Properties() -rootProject.file('.env.dev').withInputStream { env.load(it) } - -airbyte { - application { - mainClass = 'io.airbyte.api.server.ApplicationKt' - defaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] - localEnvVars = env + [ - "AIRBYTE_ROLE" : System.getenv("AIRBYTE_ROLE") ?: "undefined", - "AIRBYTE_VERSION" : env.VERSION, - "MICRONAUT_ENVIRONMENTS": "control-plane", - "SERVICE_NAME" : project.name, - "TRACKING_STRATEGY" : env.TRACKING_STRATEGY - ] as Map - } - docker { - imageName = "airbyte-api-server" - } -} - -test { - environment 'AIRBYTE_VERSION', env.VERSION - environment 'MICRONAUT_ENVIRONMENTS', 'test' - environment 'SERVICE_NAME', project.name -} diff --git a/airbyte-api-server/build.gradle.kts b/airbyte-api-server/build.gradle.kts new file mode 100644 index 00000000000..54e5d48559a --- /dev/null +++ b/airbyte-api-server/build.gradle.kts @@ -0,0 +1,93 @@ +import java.util.Properties + +plugins { + id("io.airbyte.gradle.jvm.app") + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + id("io.airbyte.gradle.docker") + kotlin("jvm") + kotlin("kapt") +} + +dependencies { + implementation("org.apache.logging.log4j:log4j-slf4j2-impl") + + kapt(platform(libs.micronaut.bom)) + kapt(libs.bundles.micronaut.annotation.processor) + + kaptTest(platform(libs.micronaut.bom)) + kaptTest(libs.bundles.micronaut.test.annotation.processor) + + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + annotationProcessor(libs.micronaut.jaxrs.processor) + + implementation(project(":airbyte-analytics")) + implementation(project(":airbyte-api")) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-config:config-models")) + implementation("com.cronutils:cron-utils:9.2.1") + implementation(libs.bundles.jackson) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.bundles.micronaut.data.jdbc) + implementation(libs.micronaut.jaxrs.server) + implementation(libs.micronaut.problem.json) + implementation(libs.micronaut.security) + + implementation(libs.sentry.java) + implementation(libs.swagger.annotations) + implementation(libs.javax.ws.rs.api) + + runtimeOnly(libs.javax.databind) + + testImplementation(libs.bundles.micronaut.test) + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + + testImplementation(project(":airbyte-test-utils")) + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.postgresql) + testImplementation(libs.platform.testcontainers.postgresql) + testImplementation(libs.mockwebserver) + testImplementation(libs.mockito.inline) + + implementation(libs.airbyte.protocol) +} + +kapt { + correctErrorTypes = true +} + +val env = Properties().apply { + load(rootProject.file(".env.dev").inputStream()) +} + +airbyte { + application { + mainClass = "io.airbyte.api.server.ApplicationKt" + defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + + @Suppress("UNCHECKED_CAST") + localEnvVars.putAll(env.toMutableMap() as Map) + localEnvVars.putAll(mapOf( + "AIRBYTE_ROLE" to (System.getenv("AIRBYTE_ROLE") ?: "undefined"), + "AIRBYTE_VERSION" to env["VERSION"].toString(), + "MICRONAUT_ENVIRONMENTS" to "control-plane", + "SERVICE_NAME" to project.name, + "TRACKING_STRATEGY" to env["TRACKING_STRATEGY"].toString(), + )) + } + docker { + imageName = "airbyte-api-server" + } +} + +tasks.named("test") { + environment(mapOf( + "AIRBYTE_VERSION" to env["VERSION"], + "MICRONAUT_ENVIRONMENTS" to "test", + "SERVICE_NAME" to project.name, + )) +} diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle index 4b32c51359f..5598422f100 100644 --- a/airbyte-api/build.gradle +++ b/airbyte-api/build.gradle @@ -158,7 +158,7 @@ import dev.failsafe.okhttp.FailsafeCall''' } apiClientFile.write(apiClientFileText) - // Update domain clients to use Failesafe + updateDomainClientsWithFailsafe('build/generated/api/client2/src/main/kotlin/io/airbyte/api/client2/generated') } } @@ -267,8 +267,11 @@ def genWorkloadApiClient = tasks.register("genWorkloadApiClient", GenerateTask) doLast { // Delete file generated by the client2 task def dir = file('build/generated/workloadapi/client/src/main/kotlin/org').deleteDir() - // Update domain clients to use Failsafe - updateDomainClientsWithFailsafe('build/generated/workloadapi/client/src/main/kotlin/io/airbyte/workload/api/client/generated') + + def generatedDomainClientsPath = 'build/generated/workloadapi/client/src/main/kotlin/io/airbyte/workload/api/client/generated' + updateDomainClientsWithFailsafe(generatedDomainClientsPath) + // the kotlin client (as opposed to the java client) doesn't include the response body in the exception message. + updateDomainClientsToIncludeHttpResponseBodyOnClientException(generatedDomainClientsPath) } dependsOn(':airbyte-workload-api-server:compileKotlin', 'genApiClient2') @@ -333,7 +336,7 @@ sourceSets { "$buildDir/generated/api/client/src/main/java", "$buildDir/generated/api/client2/src/main/kotlin", "$buildDir/generated/workloadapi/client/src/main/kotlin" - "$projectDir/src/main/java" + "$projectDir/src/main/java" } resources { srcDir "$projectDir/src/main/openapi/" @@ -373,7 +376,23 @@ private def updateDomainClientsWithFailsafe(def clientPath) { def newImports = "import dev.failsafe.RetryPolicy" domainClientFileText = domainClientFileText.replaceFirst('import ', newImports + '\nimport ') } + domainClient.write(domainClientFileText) } } -} \ No newline at end of file +} + +private def updateDomainClientsToIncludeHttpResponseBodyOnClientException(def clientPath) { + def dir = file(clientPath) + dir.eachFile { domainClient -> + if (domainClient.name.endsWith('.kt')) { + def domainClientFileText = domainClient.text + + domainClientFileText = domainClientFileText.replace( + 'throw ClientException("Client error : ${localVarError.statusCode} ${localVarError.message.orEmpty()}", localVarError.statusCode, localVarResponse)', + 'throw ClientException("Client error : ${localVarError.statusCode} ${localVarError.message.orEmpty()} ${localVarError.body ?: ""}", localVarError.statusCode, localVarResponse)') + + domainClient.write(domainClientFileText) + } + } +} diff --git a/airbyte-api/new-build.gradle.kts b/airbyte-api/new-build.gradle.kts new file mode 100644 index 00000000000..787e1135f61 --- /dev/null +++ b/airbyte-api/new-build.gradle.kts @@ -0,0 +1,384 @@ +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask + +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + id("org.openapi.generator") + kotlin("jvm") + kotlin("kapt") +} + +val specFile = "$projectDir/src/main/openapi/config.yaml" +val airbyteApiSpecFile = "$projectDir/src/main/openapi/api.yaml" +val airbyteApiSpecTemplateDirApi = "$projectDir/src/main/resources/templates/jaxrs-spec-api" +val workloadSpecFile = "$projectDir/src/main/openapi/workload-openapi.yaml" + +val genApiServer = tasks.register("generateApiServer") { + val serverOutputDir = "$buildDir/generated/api/server" + + inputs.file(specFile) + outputs.dir(serverOutputDir) + + generatorName = "jaxrs-spec" + inputSpec = specFile + outputDir = serverOutputDir + + apiPackage = "io.airbyte.api.generated" + invokerPackage = "io.airbyte.api.invoker.generated" + modelPackage = "io.airbyte.api.model.generated" + + schemaMappings = mapOf( + "OAuthConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "SourceDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "SourceConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "StreamJsonSchema" to "com.fasterxml.jackson.databind.JsonNode", + "StateBlob" to "com.fasterxml.jackson.databind.JsonNode", + "FieldSchema" to "com.fasterxml.jackson.databind.JsonNode", + "DeclarativeManifest" to "com.fasterxml.jackson.databind.JsonNode", + "SecretPersistenceConfigurationJson" to "com.fasterxml.jackson.databind.JsonNode", + ) + + generateApiDocumentation = false + + configOptions = mapOf( + "dateLibrary" to "java8", + "generatePom" to "false", + "interfaceOnly" to "true", + /*) + JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. + It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. + The below Jackson annotation(is made to only(keep non null values in serialized json. + We are not yet using nullable=true properties in our OpenApi so this is a valid(workaround at the moment to circumvent the default JAX-RS behavior described above. + Feel free to read the conversation(on https://github.com/airbytehq/airbyte/pull/13370 for more details. + */ + "additionalModelTypeAnnotations" to "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", + + // Generate separate classes for each endpoint "domain") + "useTags" to "true", + ) +} + +val genApiClient = tasks.register("generateApiClient") { + val clientOutputDir = "$buildDir/generated/api/client" + + inputs.file(specFile) + outputs.dir(clientOutputDir) + + generatorName = "java" + inputSpec = specFile + outputDir = clientOutputDir + + apiPackage = "io.airbyte.api.client.generated" + invokerPackage = "io.airbyte.api.client.invoker.generated" + modelPackage = "io.airbyte.api.client.model.generated" + + schemaMappings = mapOf( + "OAuthConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "SourceDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "SourceConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "StreamJsonSchema" to "com.fasterxml.jackson.databind.JsonNode", + "StateBlob" to "com.fasterxml.jackson.databind.JsonNode", + "FieldSchema" to "com.fasterxml.jackson.databind.JsonNode", + "SecretPersistenceConfigurationJson" to "com.fasterxml.jackson.databind.JsonNode", + ) + + library = "native" + + generateApiDocumentation = false + + configOptions = mapOf( + "dateLibrary" to "java8", + "generatePom" to "false", + "interfaceOnly" to "true", + ) +} + +val genApiClient2 = tasks.register("genApiClient2") { + val clientOutputDir = "$buildDir/generated/api/client2" + + inputs.file(specFile) + outputs.dir(clientOutputDir) + + generatorName = "kotlin" + inputSpec = specFile + outputDir = clientOutputDir + + apiPackage = "io.airbyte.api.client2.generated" + invokerPackage = "io.airbyte.api.client2.invoker.generated" + modelPackage = "io.airbyte.api.client2.model.generated" + + schemaMappings = mapOf( + "OAuthConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "SourceDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "SourceConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "StreamJsonSchema" to "com.fasterxml.jackson.databind.JsonNode", + "StateBlob" to "com.fasterxml.jackson.databind.JsonNode", + "FieldSchema" to "com.fasterxml.jackson.databind.JsonNode", + "SecretPersistenceConfigurationJson" to "com.fasterxml.jackson.databind.JsonNode", + ) + + generateApiDocumentation = false + + configOptions = mapOf( + "generatePom" to "false", + "interfaceOnly" to "true", + ) + + doLast { + /*) + * UPDATE ApiClient.kt to use Failsafe. + */ + var apiClientFile = file("build/generated/api/client2/src/main/kotlin/org/openapitools/client/infrastructure/ApiClient.kt") + var apiClientFileText = apiClientFile.readText() + + // replace class declaration) + apiClientFileText = apiClientFileText.replace( + "open class ApiClient(val baseUrl: String, val client: OkHttpClient = defaultClient) {", + "open class ApiClient(val baseUrl: String, val client: OkHttpClient = defaultClient, val policy: RetryPolicy = RetryPolicy.ofDefaults()) {") + + // replace execute call) + apiClientFileText = apiClientFileText.replace( + "val response = client.newCall(request).execute()", + """val call = client.newCall(request) + val failsafeCall = FailsafeCall.with(policy).compose(call) + val response: Response = failsafeCall.execute()""") + + // add imports if not exist) + if (!apiClientFileText.contains("import dev.failsafe.RetryPolicy")) { + val newImports = """import dev.failsafe.RetryPolicy +import dev.failsafe.okhttp.FailsafeCall""" + apiClientFileText = apiClientFileText.replaceFirst("import ", "$newImports\nimport ") + + } + apiClientFile.writeText(apiClientFileText) + + // Update domain clients to use Failesafe + updateDomainClientsWithFailsafe("build/generated/api/client2/src/main/kotlin/io/airbyte/api/client2/generated") + } +} + +val genApiDocs = tasks.register("generateApiDocs") { + val docsOutputDir = "$buildDir/generated/api/docs" + + generatorName = "html" + inputSpec = specFile + outputDir = docsOutputDir + + apiPackage = "io.airbyte.api.client.generated" + invokerPackage = "io.airbyte.api.client.invoker.generated" + modelPackage = "io.airbyte.api.client.model.generated" + + schemaMappings = mapOf( + "OAuthConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "SourceDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "SourceConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "StreamJsonSchema" to "com.fasterxml.jackson.databind.JsonNode", + "StateBlob" to "com.fasterxml.jackson.databind.JsonNode", + "FieldSchema" to "com.fasterxml.jackson.databind.JsonNode", + ) + + generateApiDocumentation = false + + configOptions = mapOf( + "dateLibrary" to "java8", + "generatePom" to "false", + "interfaceOnly" to "true", + ) +} + +val genAirbyteApiServer = tasks.register("generateAirbyteApiServer") { + val serverOutputDir = "$buildDir/generated/airbyte_api/server" + + inputs.file(airbyteApiSpecFile) + outputs.dir(serverOutputDir) + + generatorName = "jaxrs-spec" + inputSpec = airbyteApiSpecFile + outputDir = serverOutputDir + templateDir = airbyteApiSpecTemplateDirApi + + apiPackage = "io.airbyte.airbyte-api.generated" + invokerPackage = "io.airbyte.airbyte-api.invoker.generated" + modelPackage = "io.airbyte.airbyte-api.model.generated" + + generateApiDocumentation = false + + configOptions = mapOf( + "dateLibrary" to "java8", + "generatePom" to "false", + "interfaceOnly" to "true", + "returnResponse" to "true", + "useBeanValidation" to "true", + "performBeanValidation" to "true", + "additionalModelTypeAnnotations" to "@io.micronaut.core.annotation.Introspected", + "additionalEnumTypeAnnotations" to "@io.micronaut.core.annotation.Introspected", + ) + + schemaMappings = mapOf( + "SourceConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "OAuthInputConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "OAuthCredentialsConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + ) +} + +// TODO: Linked to document okhhtp +val genWorkloadApiClient = tasks.register("genWorkloadApiClient") { + val clientOutputDir = "$buildDir/generated/workloadapi/client" + + inputs.file(workloadSpecFile) + outputs.dir(clientOutputDir) + + generatorName = "kotlin" + inputSpec = workloadSpecFile + outputDir = clientOutputDir + + apiPackage = "io.airbyte.workload.api.client.generated" + invokerPackage = "io.airbyte.workload.api.client.invoker.generated" + modelPackage = "io.airbyte.workload.api.client.model.generated" + + schemaMappings = mapOf( + "OAuthConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "SourceDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "SourceConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationDefinitionSpecification" to "com.fasterxml.jackson.databind.JsonNode", + "DestinationConfiguration" to "com.fasterxml.jackson.databind.JsonNode", + "StreamJsonSchema" to "com.fasterxml.jackson.databind.JsonNode", + "StateBlob" to "com.fasterxml.jackson.databind.JsonNode", + "FieldSchema" to "com.fasterxml.jackson.databind.JsonNode", + ) + + generateApiDocumentation = false + + configOptions = mapOf( + "enumPropertyNaming" to "UPPERCASE", + "generatePom" to "false", + "interfaceOnly" to "true", + ) + + doLast { + // Delete file generated by the client2 task) + file("build/generated/workloadapi/client/src/main/kotlin/org").delete() + // Update domain clients to use Failsafe + updateDomainClientsWithFailsafe("build/generated/workloadapi/client/src/main/kotlin/io/airbyte/workload/api/client/generated") + } + + dependsOn(":airbyte-workload-api-server:compileKotlin", genApiClient2) +} + + +tasks.named("compileJava") { + dependsOn(genApiDocs, genApiClient, genApiServer, genAirbyteApiServer) +} + +kapt { + correctErrorTypes = true +} + +// uses afterEvaluate because at configuration(time, the kaptGenerateStubsKotlin task does not exist.) +afterEvaluate { + tasks.named("kaptGenerateStubsKotlin").configure { + mustRunAfter(genApiDocs, genApiClient, genApiClient2, genApiServer, genAirbyteApiServer, genWorkloadApiClient) + } +} + +tasks.named("compileKotlin") { + dependsOn(genApiClient2, tasks.named("genWorkloadApiClient")) +} + +dependencies { + annotationProcessor(libs.micronaut.openapi) + kapt(libs.micronaut.openapi) + + compileOnly(libs.v3.swagger.annotations) + kapt(libs.v3.swagger.annotations) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.commons.io) + implementation(libs.failsafe.okhttp) + implementation(libs.guava) + implementation(libs.javax.annotation.api) + implementation(libs.javax.ws.rs.api) + implementation(libs.javax.validation.api) + implementation(libs.jackson.datatype) + implementation(libs.moshi.kotlin) + implementation(libs.okhttp) + implementation(libs.openapi.jackson.databind.nullable) + implementation(libs.reactor.core) + implementation(libs.slf4j.api) + implementation(libs.swagger.annotations) + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + testImplementation(libs.junit.pioneer) + testImplementation(libs.mockk) + testImplementation(libs.kotlin.test.runner.junit5) +} + +sourceSets["main"].java { +// java { + srcDirs("$buildDir/generated/api/server/src/gen/java", + "$buildDir/generated/airbyte_api/server/src/gen/java", + "$buildDir/generated/api/client/src/main/java", + "$buildDir/generated/api/client2/src/main/kotlin", + "$buildDir/generated/workloadapi/client/src/main/kotlin", + "$projectDir/src/main/java", + ) + } +// resources { +// srcDir("$projectDir/src/main/openapi/") +// } +// } +//} +sourceSets["main"].resources { + srcDir("$projectDir/src/main/openapi/") +} + +tasks.withType().configureEach { + options.compilerArgs.add("-parameters") +} + +airbyte { + spotless { + excludes = listOf( + "src/main/openapi/workload-openapi.yaml", + "$buildDir/generated/**", + ) + } +} + +fun updateDomainClientsWithFailsafe(clientPath:String) { + /* + * UPDATE domain clients to use Failsafe. + */ + val dir = file(clientPath) + dir.walk().forEach { domainClient -> +// println("looking at file $domainClient") + if (domainClient.name.endsWith(".kt")) { + var domainClientFileText = domainClient.readText() + + // replace class declaration + domainClientFileText = domainClientFileText.replace( + """class (\S+)\(basePath: kotlin.String = defaultBasePath, client: OkHttpClient = ApiClient.defaultClient\) : ApiClient\(basePath, client\)""".toRegex(), + "class $1(basePath: kotlin.String = defaultBasePath, client: OkHttpClient = ApiClient.defaultClient, policy: RetryPolicy = RetryPolicy.ofDefaults()) : ApiClient(basePath, client, policy)" + ) + + // add imports if not exist) + if(!domainClientFileText.contains("import dev.failsafe.RetryPolicy")) { + val newImports = "import dev.failsafe.RetryPolicy" + domainClientFileText = domainClientFileText.replaceFirst("import ", "$newImports\nimport ") + } + domainClient.writeText(domainClientFileText) + } + } +} diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 53cf32521cc..e50a329b01a 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -6447,6 +6447,8 @@ components: type: boolean orgLevelBilling: type: boolean + email: + type: string OrganizationCreateRequestBody: type: object required: diff --git a/airbyte-api/src/main/openapi/workload-openapi.yaml b/airbyte-api/src/main/openapi/workload-openapi.yaml index ee8475b377d..587eb8cda97 100644 --- a/airbyte-api/src/main/openapi/workload-openapi.yaml +++ b/airbyte-api/src/main/openapi/workload-openapi.yaml @@ -22,8 +22,16 @@ paths: description: Success "404": description: Workload with given id was not found. + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' "410": description: "Workload is in terminal state, it cannot be cancelled." + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' /api/v1/workload/claim: put: tags: @@ -45,8 +53,16 @@ paths: $ref: '#/components/schemas/ClaimResponse' "404": description: Workload with given id was not found. + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' "410": description: "Workload is in terminal state, it cannot be claimed." + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' /api/v1/workload/create: post: tags: @@ -61,8 +77,12 @@ paths: responses: "204": description: Successfully created workload - "304": + "409": description: Workload with given workload id already exists. + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' /api/v1/workload/failure: put: tags: @@ -79,8 +99,16 @@ paths: description: Success "404": description: Workload with given id was not found. + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' "410": description: "Workload is not in an active state, it cannot be failed." + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' /api/v1/workload/heartbeat: put: tags: @@ -97,9 +125,17 @@ paths: description: Successfully heartbeated "404": description: Workload with given id was not found. + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' "410": description: Workload should stop because it is no longer expected to be running. + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' /api/v1/workload/list: post: tags: @@ -134,8 +170,16 @@ paths: description: Success "404": description: Workload with given id was not found. + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' "410": description: "Workload is not in pending status, it can't be set to running." + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' /api/v1/workload/success: put: tags: @@ -152,8 +196,16 @@ paths: description: Success "404": description: Workload with given id was not found. + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' "410": description: "Workload is not in an active state, it cannot be succeeded." + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' /api/v1/workload/{workloadId}: get: tags: @@ -175,6 +227,10 @@ paths: $ref: '#/components/schemas/Workload' "404": description: Workload with given id was not found. + content: + application/json: + schema: + $ref: '#/components/schemas/KnownExceptionInfo' components: schemas: ClaimResponse: @@ -184,8 +240,16 @@ components: properties: claimed: type: boolean + KnownExceptionInfo: + required: + - message + type: object + properties: + message: + type: string Workload: required: + - geography - id - inputPayload - labels @@ -209,6 +273,8 @@ components: type: string logPath: type: string + geography: + type: string WorkloadCancelRequest: required: - reason @@ -234,6 +300,7 @@ components: type: string WorkloadCreateRequest: required: + - geography - labels - logPath - workloadId @@ -250,6 +317,8 @@ components: type: string logPath: type: string + geography: + type: string WorkloadFailureRequest: required: - workloadId diff --git a/airbyte-api/src/test/kotlin/io/airbyte/api/client/WorkloadApiTest.kt b/airbyte-api/src/test/kotlin/io/airbyte/api/client/WorkloadApiTest.kt new file mode 100644 index 00000000000..b46c9814dc9 --- /dev/null +++ b/airbyte-api/src/test/kotlin/io/airbyte/api/client/WorkloadApiTest.kt @@ -0,0 +1,59 @@ +package io.airbyte.api.client + +import dev.failsafe.RetryPolicy +import io.airbyte.workload.api.client.generated.WorkloadApi +import io.airbyte.workload.api.client.model.generated.WorkloadCancelRequest +import io.mockk.every +import io.mockk.mockk +import io.mockk.spyk +import okhttp3.OkHttpClient +import okhttp3.Response +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.openapitools.client.infrastructure.ClientError +import org.openapitools.client.infrastructure.ClientException + +/** + * [WorkloadApi] is a generated class using OpenAPI Generator. We are making some changes to the generated class + * in a gradle task. This test is meant to test these changes. + */ +class WorkloadApiTest { + companion object { + const val MESSAGE = "message" + const val BODY = "body" + const val STATUS_CODE = 400 + const val BASE_PATH = "basepath" + } + + @Test + fun `test client exception includes http response body`() { + val client: OkHttpClient = mockk() + val policy: RetryPolicy = mockk(relaxed = true) + val workloadApi = spyk(WorkloadApi(BASE_PATH, client, policy)) + + every { + workloadApi.workloadCancelWithHttpInfo(any()) + } returns ClientError(MESSAGE, BODY, STATUS_CODE, mapOf()) + + val exception = assertThrows { workloadApi.workloadCancel(WorkloadCancelRequest("workloadId", "reason", "source")) } + assertTrue(exception.message!!.contains(MESSAGE)) + assertTrue(exception.message!!.contains(BODY)) + assertTrue(exception.message!!.contains(STATUS_CODE.toString())) + } + + @Test + fun `test client exception null http response body`() { + val client: OkHttpClient = mockk() + val policy: RetryPolicy = mockk(relaxed = true) + val workloadApi = spyk(WorkloadApi(BASE_PATH, client, policy)) + + every { + workloadApi.workloadCancelWithHttpInfo(any()) + } returns ClientError(MESSAGE, null, STATUS_CODE, mapOf()) + + val exception = assertThrows { workloadApi.workloadCancel(WorkloadCancelRequest("workloadId", "reason", "source")) } + assertTrue(exception.message!!.contains(MESSAGE)) + assertTrue(exception.message!!.contains(STATUS_CODE.toString())) + } +} diff --git a/airbyte-bootloader/build.gradle b/airbyte-bootloader/build.gradle deleted file mode 100644 index b0b4a3a669c..00000000000 --- a/airbyte-bootloader/build.gradle +++ /dev/null @@ -1,73 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.app" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -configurations.all { - resolutionStrategy { - force libs.flyway.core, libs.jooq - } -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.flyway.core - implementation libs.jooq - implementation libs.guava - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-micronaut') - implementation project(':airbyte-config:init') - implementation project(':airbyte-config:specs') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(":airbyte-config:config-secrets") - implementation project(':airbyte-data') - implementation project(':airbyte-db:db-lib') - implementation project(":airbyte-json-validation") - implementation project(":airbyte-featureflag") - implementation libs.airbyte.protocol - implementation project(':airbyte-persistence:job-persistence') - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - - testImplementation libs.bundles.micronaut.test - testImplementation libs.bundles.junit - testImplementation libs.junit.jupiter.system.stubs - testImplementation libs.platform.testcontainers.postgresql - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - testImplementation libs.junit.pioneer - -} - -Properties env = new Properties() -rootProject.file('.env.dev').withInputStream { env.load(it) } - -airbyte { - application { - mainClass = 'io.airbyte.bootloader.Application' - defaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] - localEnvVars = env + [ - "AIRBYTE_ROLE" : System.getenv("AIRBYTE_ROLE") ?: "undefined", - "AIRBYTE_VERSION": env.VERSION, - "DATABASE_URL" : 'jdbc:postgresql://localhost:5432/airbyte' - ] as Map - } - - docker { - imageName = "bootloader" - } -} diff --git a/airbyte-bootloader/build.gradle.kts b/airbyte-bootloader/build.gradle.kts new file mode 100644 index 00000000000..622db523b09 --- /dev/null +++ b/airbyte-bootloader/build.gradle.kts @@ -0,0 +1,78 @@ +import java.util.Properties + +plugins { + id("io.airbyte.gradle.jvm.app") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +configurations.all { + resolutionStrategy { + force(libs.flyway.core, libs.jooq) + } +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.flyway.core) + implementation(libs.jooq) + implementation(libs.guava) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-micronaut")) + implementation(project(":airbyte-config:init")) + implementation(project(":airbyte-config:specs")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-config:config-secrets")) + implementation(project(":airbyte-data")) + implementation(project(":airbyte-db:db-lib")) + implementation(project(":airbyte-json-validation")) + implementation(project(":airbyte-featureflag")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-persistence:job-persistence")) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.bundles.junit) + testImplementation(libs.junit.jupiter.system.stubs) + testImplementation(libs.platform.testcontainers.postgresql) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + testImplementation(libs.junit.pioneer) + +} + +val env = Properties().apply { + load(rootProject.file(".env.dev").inputStream()) +} + +airbyte { + application { + mainClass = "io.airbyte.bootloader.Application" + defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + @Suppress("UNCHECKED_CAST") + localEnvVars.putAll(env.toMutableMap() as Map) + localEnvVars.putAll(mapOf( + "AIRBYTE_ROLE" to (System.getenv("AIRBYTE_ROLE") ?: "undefined"), + "AIRBYTE_VERSION" to env["VERSION"].toString(), + "DATABASE_URL" to "jdbc:postgresql://localhost:5432/airbyte", + )) + } + + docker { + imageName = "bootloader" + } +} diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java index 606225cd4d1..6741eda8613 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java @@ -15,7 +15,6 @@ import io.airbyte.config.init.PostLoadExecutor; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.OrganizationPersistence; -import io.airbyte.config.persistence.UserPersistence; import io.airbyte.config.persistence.WorkspacePersistence; import io.airbyte.db.init.DatabaseInitializationException; import io.airbyte.db.init.DatabaseInitializer; @@ -209,8 +208,13 @@ private void createWorkspaceIfNoneExists(final ConfigRepository configRepository final UUID workspaceId = UUID.randomUUID(); final StandardWorkspace workspace = new StandardWorkspace() .withWorkspaceId(workspaceId) - // attach this new workspace to the default User which should always exist at this point. - .withCustomerId(UserPersistence.DEFAULT_USER_ID) + // NOTE: we made a change to set this to the default User ID. It was reverted back to a random UUID + // because we discovered that our Segment Tracking Client uses distinct customer IDs to track the + // number of OSS instances deployed. this is flawed because now, a single OSS instance can have + // multiple workspaces. The long term fix is to update our analytics stack to use an instance-level + // identifier, like deploymentId, instead of a workspace-level identifier. For a quick fix though, + // we're reverting back to a randomized customer ID for the default workspace. + .withCustomerId(UUID.randomUUID()) .withName(WorkspacePersistence.DEFAULT_WORKSPACE_NAME) .withSlug(workspaceId.toString()) .withInitialSetupComplete(false) diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index 79ce261d146..70f97dec307 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -95,7 +95,7 @@ class BootloaderTest { // ⚠️ This line should change with every new migration to show that you meant to make a new // migration to the prod database - private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.33.006"; + private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.33.007"; private static final String CURRENT_JOBS_MIGRATION_VERSION = "0.50.4.001"; private static final String CDK_VERSION = "1.2.3"; diff --git a/airbyte-commons-auth/build.gradle b/airbyte-commons-auth/build.gradle deleted file mode 100644 index a8dfff5f59b..00000000000 --- a/airbyte-commons-auth/build.gradle +++ /dev/null @@ -1,45 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - id "org.jetbrains.kotlin.jvm" - id "org.jetbrains.kotlin.kapt" -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - kapt libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.failsafe.okhttp - implementation libs.kotlin.logging - implementation libs.okhttp - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation project(':airbyte-commons') - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - - testImplementation libs.bundles.micronaut.test - testImplementation libs.mockito.inline - testImplementation libs.mockk -} - -test { - maxHeapSize = '2g' -} - -// The DuplicatesStrategy will be required while this module is mixture of kotlin and java _with_ lombok dependencies. -// Kapt, by default, runs all annotation processors and disables annotation processing by javac, however -// this default behavior breaks the lombok java annotation processor. To avoid lombok breaking, kapt has -// keepJavacAnnotationProcessors enabled, which causes duplicate META-INF files to be generated. -// Once lombok has been removed, this can also be removed. -tasks.withType(Jar).configureEach { - duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} diff --git a/airbyte-commons-auth/build.gradle.kts b/airbyte-commons-auth/build.gradle.kts new file mode 100644 index 00000000000..cd38a5109fe --- /dev/null +++ b/airbyte-commons-auth/build.gradle.kts @@ -0,0 +1,45 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + kotlin("jvm") + kotlin("kapt") +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + kapt(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.failsafe.okhttp) + implementation(libs.kotlin.logging) + implementation(libs.okhttp) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + implementation(project(":airbyte-commons")) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.mockito.inline) + testImplementation(libs.mockk) +} + +tasks.named("test") { + maxHeapSize = "2g" +} + +// The DuplicatesStrategy will be required while this module is mixture of kotlin and java _with_ lombok dependencies. +// Kapt, by default, runs all annotation processors and disables annotation processing by javac, however +// this default behavior breaks the lombok java annotation processor. To avoid lombok breaking, kapt has +// keepJavacAnnotationProcessors enabled, which causes duplicate META-INF files to be generated. +// Once lombok has been removed, this can also be removed. +tasks.withType().configureEach { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} diff --git a/airbyte-commons-converters/build.gradle b/airbyte-commons-converters/build.gradle deleted file mode 100644 index bdc8844e3f8..00000000000 --- a/airbyte-commons-converters/build.gradle +++ /dev/null @@ -1,36 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - - implementation libs.apache.commons.text - - implementation project(':airbyte-api') - implementation project(':airbyte-commons') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-json-validation') - implementation project(':airbyte-persistence:job-persistence') - implementation libs.airbyte.protocol - implementation libs.guava - implementation libs.slf4j.api - implementation libs.bundles.datadog - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - testAnnotationProcessor libs.jmh.annotations - - testImplementation libs.bundles.micronaut.test - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer -} diff --git a/airbyte-commons-converters/build.gradle.kts b/airbyte-commons-converters/build.gradle.kts new file mode 100644 index 00000000000..c5fb7b9f18b --- /dev/null +++ b/airbyte-commons-converters/build.gradle.kts @@ -0,0 +1,36 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + + implementation(libs.apache.commons.text) + + implementation(project(":airbyte-api")) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-json-validation")) + implementation(project(":airbyte-persistence:job-persistence")) + implementation(libs.airbyte.protocol) + implementation(libs.guava) + implementation(libs.slf4j.api) + implementation(libs.bundles.datadog) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + testAnnotationProcessor(libs.jmh.annotations) + + testImplementation(libs.bundles.micronaut.test) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) +} diff --git a/airbyte-commons-license/build.gradle b/airbyte-commons-license/build.gradle deleted file mode 100644 index cd78c944d64..00000000000 --- a/airbyte-commons-license/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.guava - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-micronaut') - implementation project(':airbyte-config:config-models') - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - - testImplementation libs.bundles.micronaut.test - testImplementation libs.mockito.inline -} - -test { - maxHeapSize = '2g' -} diff --git a/airbyte-commons-license/build.gradle.kts b/airbyte-commons-license/build.gradle.kts new file mode 100644 index 00000000000..a3b7c8f1aa1 --- /dev/null +++ b/airbyte-commons-license/build.gradle.kts @@ -0,0 +1,31 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.guava) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-micronaut")) + implementation(project(":airbyte-config:config-models")) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.mockito.inline) +} + +tasks.named("test") { + maxHeapSize = "2g" +} diff --git a/airbyte-commons-micronaut/build.gradle b/airbyte-commons-micronaut/build.gradle deleted file mode 100644 index b31f7e509c8..00000000000 --- a/airbyte-commons-micronaut/build.gradle +++ /dev/null @@ -1,30 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.micronaut.security - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation project(':airbyte-commons') - implementation project(':airbyte-config:config-models') - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - - testImplementation libs.bundles.micronaut.test - testImplementation libs.mockito.inline -} - -test { - maxHeapSize = '2g' -} diff --git a/airbyte-commons-micronaut/build.gradle.kts b/airbyte-commons-micronaut/build.gradle.kts new file mode 100644 index 00000000000..171d9ce7a18 --- /dev/null +++ b/airbyte-commons-micronaut/build.gradle.kts @@ -0,0 +1,30 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.micronaut.security) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-config:config-models")) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.mockito.inline) +} + +tasks.named("test") { + maxHeapSize = "2g" +} diff --git a/airbyte-commons-protocol/build.gradle b/airbyte-commons-protocol/build.gradle deleted file mode 100644 index f3c9af15632..00000000000 --- a/airbyte-commons-protocol/build.gradle +++ /dev/null @@ -1,27 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - implementation libs.bundles.micronaut.annotation - testImplementation libs.bundles.micronaut.test - - implementation project(':airbyte-commons') - implementation libs.airbyte.protocol - implementation project(':airbyte-json-validation') - implementation libs.guava - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation libs.bundles.jackson - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer -} diff --git a/airbyte-commons-protocol/build.gradle.kts b/airbyte-commons-protocol/build.gradle.kts new file mode 100644 index 00000000000..2625ec8a3c0 --- /dev/null +++ b/airbyte-commons-protocol/build.gradle.kts @@ -0,0 +1,27 @@ +plugins { + id("io.airbyte.gradle.jvm") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(libs.bundles.micronaut.annotation.processor) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + + implementation(libs.bundles.micronaut.annotation) + testImplementation(libs.bundles.micronaut.test) + + implementation(project(":airbyte-commons")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-json-validation")) + implementation(libs.guava) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + implementation(libs.bundles.jackson) + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/CatalogTransforms.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/CatalogTransforms.java index 7182e9fc697..608d7553eb1 100644 --- a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/CatalogTransforms.java +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/CatalogTransforms.java @@ -6,9 +6,11 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.protocol.models.SyncMode; +import java.util.Iterator; import java.util.List; /** @@ -22,7 +24,9 @@ public class CatalogTransforms { public static void updateCatalogForReset( final List streamsToReset, final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - configuredAirbyteCatalog.getStreams().forEach(configuredAirbyteStream -> { + Iterator iterator = configuredAirbyteCatalog.getStreams().iterator(); + while (iterator.hasNext()) { + ConfiguredAirbyteStream configuredAirbyteStream = iterator.next(); final StreamDescriptor streamDescriptor = CatalogHelpers.extractDescriptor(configuredAirbyteStream); if (streamsToReset.contains(streamDescriptor)) { // The Reset Source will emit no record messages for any streams, so setting the destination sync @@ -33,12 +37,11 @@ public static void updateCatalogForReset( configuredAirbyteStream.setSyncMode(SyncMode.FULL_REFRESH); configuredAirbyteStream.setDestinationSyncMode(DestinationSyncMode.OVERWRITE); } else { - // Set streams that are not being reset to APPEND so that they are not modified in the destination - if (configuredAirbyteStream.getDestinationSyncMode() == DestinationSyncMode.OVERWRITE) { - configuredAirbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); - } + // Remove the streams that are not being reset. + iterator.remove(); } - }); + } + } } diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/CatalogTransformsTest.java b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/CatalogTransformsTest.java index fc306e0d99d..a9566d765ff 100644 --- a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/CatalogTransformsTest.java +++ b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/CatalogTransformsTest.java @@ -5,6 +5,7 @@ package io.airbyte.commons.protocol; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; @@ -52,14 +53,8 @@ void testResetCatalogSyncModeReplacementMultipleStreams() throws IOException { streamDescriptor, ConfiguredAirbyteStream::getDestinationSyncMode)); assertEquals(SyncMode.FULL_REFRESH, findStreamSyncMode(configuredAirbyteCatalog, streamDescriptor, ConfiguredAirbyteStream::getSyncMode)); - assertEquals(DestinationSyncMode.APPEND, findStreamSyncMode(configuredAirbyteCatalog, - otherStreamDescriptor, ConfiguredAirbyteStream::getDestinationSyncMode)); - assertEquals(SyncMode.INCREMENTAL, findStreamSyncMode(configuredAirbyteCatalog, - otherStreamDescriptor, ConfiguredAirbyteStream::getSyncMode)); - assertEquals(DestinationSyncMode.APPEND_DEDUP, findStreamSyncMode(configuredAirbyteCatalog, - otherStreamDescriptor2, ConfiguredAirbyteStream::getDestinationSyncMode)); - assertEquals(SyncMode.INCREMENTAL, findStreamSyncMode(configuredAirbyteCatalog, - otherStreamDescriptor2, ConfiguredAirbyteStream::getSyncMode)); + assertFalse(contains(configuredAirbyteCatalog, otherStreamDescriptor)); + assertFalse(contains(configuredAirbyteCatalog, otherStreamDescriptor2)); } private boolean isMatch(final ConfiguredAirbyteStream stream, final StreamDescriptor expected) { @@ -77,4 +72,13 @@ private T findStreamSyncMode(final ConfiguredAirbyteCatalog configuredAirbyt .findFirst().get(); } + private boolean contains(final ConfiguredAirbyteCatalog configuredAirbyteCatalog, + final StreamDescriptor match) { + return configuredAirbyteCatalog.getStreams() + .stream() + .filter(s -> isMatch(s, match)) + .findFirst() + .isPresent(); + } + } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java index 61fd83bfccb..c87be6e8dfe 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java @@ -23,6 +23,8 @@ import io.airbyte.api.model.generated.ConnectionAutoPropagateResult; import io.airbyte.api.model.generated.ConnectionAutoPropagateSchemaChange; import io.airbyte.api.model.generated.ConnectionCreate; +import io.airbyte.api.model.generated.ConnectionDataHistoryReadItem; +import io.airbyte.api.model.generated.ConnectionDataHistoryRequestBody; import io.airbyte.api.model.generated.ConnectionRead; import io.airbyte.api.model.generated.ConnectionReadList; import io.airbyte.api.model.generated.ConnectionSearch; @@ -66,6 +68,7 @@ import io.airbyte.config.Geography; import io.airbyte.config.JobConfig; import io.airbyte.config.JobConfig.ConfigType; +import io.airbyte.config.JobOutput; import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; import io.airbyte.config.Schedule; import io.airbyte.config.ScheduleData; @@ -91,6 +94,7 @@ import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.persistence.job.models.Attempt; +import io.airbyte.persistence.job.models.AttemptWithJobInfo; import io.airbyte.persistence.job.models.Job; import io.airbyte.persistence.job.models.JobStatus; import io.airbyte.persistence.job.models.JobWithStatusAndTimestamp; @@ -102,8 +106,12 @@ import jakarta.inject.Singleton; import java.io.IOException; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -171,6 +179,99 @@ public ConnectionsHandler( this.maxFailedJobsInARowBeforeConnectionDisable = maxFailedJobsInARowBeforeConnectionDisable; } + /** + * Modifies the given StandardSync by applying changes from a partially-filled ConnectionUpdate + * patch. Any fields that are null in the patch will be left unchanged. + */ + private static void applyPatchToStandardSync(final StandardSync sync, final ConnectionUpdate patch) throws JsonValidationException { + // update the sync's schedule using the patch's scheduleType and scheduleData. validations occur in + // the helper to ensure both fields + // make sense together. + if (patch.getScheduleType() != null) { + ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(sync, patch.getScheduleType(), patch.getScheduleData()); + } + + // the rest of the fields are straightforward to patch. If present in the patch, set the field to + // the value + // in the patch. Otherwise, leave the field unchanged. + + if (patch.getSyncCatalog() != null) { + validateCatalogDoesntContainDuplicateStreamNames(patch.getSyncCatalog()); + sync.setCatalog(CatalogConverter.toConfiguredProtocol(patch.getSyncCatalog())); + sync.withFieldSelectionData(CatalogConverter.getFieldSelectionData(patch.getSyncCatalog())); + } + + if (patch.getName() != null) { + sync.setName(patch.getName()); + } + + if (patch.getNamespaceDefinition() != null) { + sync.setNamespaceDefinition(Enums.convertTo(patch.getNamespaceDefinition(), NamespaceDefinitionType.class)); + } + + if (patch.getNamespaceFormat() != null) { + sync.setNamespaceFormat(patch.getNamespaceFormat()); + } + + if (patch.getPrefix() != null) { + sync.setPrefix(patch.getPrefix()); + } + + if (patch.getOperationIds() != null) { + sync.setOperationIds(patch.getOperationIds()); + } + + if (patch.getStatus() != null) { + sync.setStatus(ApiPojoConverters.toPersistenceStatus(patch.getStatus())); + } + + if (patch.getSourceCatalogId() != null) { + sync.setSourceCatalogId(patch.getSourceCatalogId()); + } + + if (patch.getResourceRequirements() != null) { + sync.setResourceRequirements(ApiPojoConverters.resourceRequirementsToInternal(patch.getResourceRequirements())); + } + + if (patch.getGeography() != null) { + sync.setGeography(ApiPojoConverters.toPersistenceGeography(patch.getGeography())); + } + + if (patch.getBreakingChange() != null) { + sync.setBreakingChange(patch.getBreakingChange()); + } + + if (patch.getNotifySchemaChanges() != null) { + sync.setNotifySchemaChanges(patch.getNotifySchemaChanges()); + } + + if (patch.getNotifySchemaChangesByEmail() != null) { + sync.setNotifySchemaChangesByEmail(patch.getNotifySchemaChangesByEmail()); + } + + if (patch.getNonBreakingChangesPreference() != null) { + sync.setNonBreakingChangesPreference(ApiPojoConverters.toPersistenceNonBreakingChangesPreference(patch.getNonBreakingChangesPreference())); + } + } + + private static String getFrequencyStringFromScheduleType(final ScheduleType scheduleType, final ScheduleData scheduleData) { + switch (scheduleType) { + case MANUAL -> { + return "manual"; + } + case BASIC_SCHEDULE -> { + return TimeUnit.SECONDS.toMinutes(ScheduleHelpers.getIntervalInSecond(scheduleData.getBasicSchedule())) + " min"; + } + case CRON -> { + // TODO(https://github.com/airbytehq/airbyte/issues/2170): consider something more detailed. + return "cron"; + } + default -> { + throw new RuntimeException("Unexpected schedule type"); + } + } + } + public InternalOperationResult autoDisableConnection(final UUID connectionId) throws JsonValidationException, IOException, ConfigNotFoundException { return autoDisableConnection(connectionId, Instant.now()); @@ -520,81 +621,6 @@ public ConnectionRead updateConnection(final ConnectionUpdate connectionPatch) return updatedRead; } - /** - * Modifies the given StandardSync by applying changes from a partially-filled ConnectionUpdate - * patch. Any fields that are null in the patch will be left unchanged. - */ - private static void applyPatchToStandardSync(final StandardSync sync, final ConnectionUpdate patch) throws JsonValidationException { - // update the sync's schedule using the patch's scheduleType and scheduleData. validations occur in - // the helper to ensure both fields - // make sense together. - if (patch.getScheduleType() != null) { - ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(sync, patch.getScheduleType(), patch.getScheduleData()); - } - - // the rest of the fields are straightforward to patch. If present in the patch, set the field to - // the value - // in the patch. Otherwise, leave the field unchanged. - - if (patch.getSyncCatalog() != null) { - validateCatalogDoesntContainDuplicateStreamNames(patch.getSyncCatalog()); - sync.setCatalog(CatalogConverter.toConfiguredProtocol(patch.getSyncCatalog())); - sync.withFieldSelectionData(CatalogConverter.getFieldSelectionData(patch.getSyncCatalog())); - } - - if (patch.getName() != null) { - sync.setName(patch.getName()); - } - - if (patch.getNamespaceDefinition() != null) { - sync.setNamespaceDefinition(Enums.convertTo(patch.getNamespaceDefinition(), NamespaceDefinitionType.class)); - } - - if (patch.getNamespaceFormat() != null) { - sync.setNamespaceFormat(patch.getNamespaceFormat()); - } - - if (patch.getPrefix() != null) { - sync.setPrefix(patch.getPrefix()); - } - - if (patch.getOperationIds() != null) { - sync.setOperationIds(patch.getOperationIds()); - } - - if (patch.getStatus() != null) { - sync.setStatus(ApiPojoConverters.toPersistenceStatus(patch.getStatus())); - } - - if (patch.getSourceCatalogId() != null) { - sync.setSourceCatalogId(patch.getSourceCatalogId()); - } - - if (patch.getResourceRequirements() != null) { - sync.setResourceRequirements(ApiPojoConverters.resourceRequirementsToInternal(patch.getResourceRequirements())); - } - - if (patch.getGeography() != null) { - sync.setGeography(ApiPojoConverters.toPersistenceGeography(patch.getGeography())); - } - - if (patch.getBreakingChange() != null) { - sync.setBreakingChange(patch.getBreakingChange()); - } - - if (patch.getNotifySchemaChanges() != null) { - sync.setNotifySchemaChanges(patch.getNotifySchemaChanges()); - } - - if (patch.getNotifySchemaChangesByEmail() != null) { - sync.setNotifySchemaChangesByEmail(patch.getNotifySchemaChangesByEmail()); - } - - if (patch.getNonBreakingChangesPreference() != null) { - sync.setNonBreakingChangesPreference(ApiPojoConverters.toPersistenceNonBreakingChangesPreference(patch.getNonBreakingChangesPreference())); - } - } - private void validateConnectionPatch(final WorkspaceHelper workspaceHelper, final StandardSync persistedSync, final ConnectionUpdate patch) { // sanity check that we're updating the right connection Preconditions.checkArgument(persistedSync.getConnectionId().equals(patch.getConnectionId())); @@ -827,24 +853,6 @@ private ConnectionRead buildConnectionRead(final UUID connectionId) return ApiPojoConverters.internalToConnectionRead(standardSync); } - private static String getFrequencyStringFromScheduleType(final ScheduleType scheduleType, final ScheduleData scheduleData) { - switch (scheduleType) { - case MANUAL -> { - return "manual"; - } - case BASIC_SCHEDULE -> { - return TimeUnit.SECONDS.toMinutes(ScheduleHelpers.getIntervalInSecond(scheduleData.getBasicSchedule())) + " min"; - } - case CRON -> { - // TODO(https://github.com/airbytehq/airbyte/issues/2170): consider something more detailed. - return "cron"; - } - default -> { - throw new RuntimeException("Unexpected schedule type"); - } - } - } - public ConnectionReadList listConnectionsForWorkspaces(final ListConnectionsForWorkspacesRequestBody listConnectionsForWorkspacesRequestBody) throws IOException { @@ -885,22 +893,23 @@ public ConnectionReadList listConnectionsForActorDefinition(final ActorDefinitio } public List getConnectionStatuses( - ConnectionStatusesRequestBody connectionStatusesRequestBody) + final ConnectionStatusesRequestBody connectionStatusesRequestBody) throws IOException, JsonValidationException, ConfigNotFoundException { - List connectionIds = connectionStatusesRequestBody.getConnectionIds(); - List result = new ArrayList<>(); - for (UUID connectionId : connectionIds) { - List jobs = jobPersistence.listJobs(Set.of(JobConfig.ConfigType.SYNC, JobConfig.ConfigType.RESET_CONNECTION), connectionId.toString(), + final List connectionIds = connectionStatusesRequestBody.getConnectionIds(); + final List result = new ArrayList<>(); + for (final UUID connectionId : connectionIds) { + final List jobs = jobPersistence.listJobs(Set.of(JobConfig.ConfigType.SYNC, JobConfig.ConfigType.RESET_CONNECTION), + connectionId.toString(), maxJobLookback); - boolean isRunning = jobs.stream().anyMatch(job -> JobStatus.NON_TERMINAL_STATUSES.contains(job.getStatus())); + final boolean isRunning = jobs.stream().anyMatch(job -> JobStatus.NON_TERMINAL_STATUSES.contains(job.getStatus())); - Optional lastJob = jobs.stream().filter(job -> JobStatus.TERMINAL_STATUSES.contains(job.getStatus())).findFirst(); - Optional lastSyncStatus = lastJob.map(job -> job.getStatus()); + final Optional lastJob = jobs.stream().filter(job -> JobStatus.TERMINAL_STATUSES.contains(job.getStatus())).findFirst(); + final Optional lastSyncStatus = lastJob.map(job -> job.getStatus()); - Optional lastSuccessfulJob = jobs.stream().filter(job -> job.getStatus() == JobStatus.SUCCEEDED).findFirst(); - Optional lastSuccessTimestamp = lastSuccessfulJob.map(job -> job.getUpdatedAtInSecond()); + final Optional lastSuccessfulJob = jobs.stream().filter(job -> job.getStatus() == JobStatus.SUCCEEDED).findFirst(); + final Optional lastSuccessTimestamp = lastSuccessfulJob.map(job -> job.getUpdatedAtInSecond()); - ConnectionStatusRead connectionStatus = new ConnectionStatusRead() + final ConnectionStatusRead connectionStatus = new ConnectionStatusRead() .connectionId(connectionId) .isRunning(isRunning) .lastSyncJobStatus(Enums.convertTo(lastSyncStatus.orElse(null), @@ -908,7 +917,7 @@ public List getConnectionStatuses( .lastSuccessfulSync(lastSuccessTimestamp.orElse(null)) .nextSync(null) .isLastCompletedJobReset(lastJob.map(job -> job.getConfigType() == ConfigType.RESET_CONNECTION).orElse(false)); - Optional failureType = + final Optional failureType = lastJob.flatMap(Job::getLastFailedAttempt) .flatMap(Attempt::getFailureSummary) .flatMap(s -> s.getFailures().stream().findFirst()) @@ -922,6 +931,65 @@ public List getConnectionStatuses( return result; } + /** + * Returns bytes committed per day for the given connection for the last 30 days in the given + * timezone. + * + * @param connectionDataHistoryRequestBody the connectionId and timezone string + * @return list of ConnectionDataHistoryReadItems (timestamp and bytes committed) + */ + public List getConnectionDataHistory(final ConnectionDataHistoryRequestBody connectionDataHistoryRequestBody) + throws IOException { + + // Start time in designated timezone + final ZonedDateTime endTimeInUserTimeZone = Instant.now().atZone(ZoneId.of(connectionDataHistoryRequestBody.getTimezone())); + final ZonedDateTime startTimeInUserTimeZone = endTimeInUserTimeZone.minusDays(30); + // Convert start time to UTC (since that's what the database uses) + final Instant startTimeInUTC = startTimeInUserTimeZone.toInstant(); + + final List attempts = jobPersistence.listAttemptsForConnectionAfterTimestamp( + connectionDataHistoryRequestBody.getConnectionId(), + ConfigType.SYNC, + startTimeInUTC); + + // we want an entry per day - even if it's empty + final Map connectionDataHistoryReadItemsByDate = new HashMap<>(); + final LocalDate startDate = startTimeInUserTimeZone.toLocalDate(); + final LocalDate endDate = endTimeInUserTimeZone.toLocalDate(); + for (LocalDate date = startDate; !date.isAfter(endDate); date = date.plusDays(1)) { + connectionDataHistoryReadItemsByDate.put(date, new ConnectionDataHistoryReadItem() + .timestamp(Math.toIntExact(date.atStartOfDay(ZoneId.of(connectionDataHistoryRequestBody.getTimezone())).toEpochSecond())) + .bytes(0)); + } + + for (final AttemptWithJobInfo attempt : attempts) { + final Optional endedAtOptional = attempt.getAttempt().getEndedAtInSecond(); + + if (endedAtOptional.isPresent()) { + // Convert the endedAt timestamp from the database to the designated timezone + final Instant attemptEndedAt = Instant.ofEpochSecond(endedAtOptional.get()); + final LocalDate attemptDateInUserTimeZone = attemptEndedAt.atZone(ZoneId.of(connectionDataHistoryRequestBody.getTimezone())) + .toLocalDate(); + + // Merge it with the bytes synced from the attempt + int bytesSynced = 0; + final Optional attemptOutput = attempt.getAttempt().getOutput(); + if (attemptOutput.isPresent()) { + bytesSynced = Math.toIntExact(attemptOutput.get().getSync().getStandardSyncSummary().getTotalStats().getBytesCommitted()); + } + + // Update the bytes synced for the corresponding day + final ConnectionDataHistoryReadItem existingItem = connectionDataHistoryReadItemsByDate.get(attemptDateInUserTimeZone); + existingItem.setBytes(existingItem.getBytes() + bytesSynced); + } + } + + // Sort the results by date + return connectionDataHistoryReadItemsByDate.values().stream() + .sorted(Comparator.comparing(ConnectionDataHistoryReadItem::getTimestamp)) + .collect(Collectors.toList()); + } + public ConnectionAutoPropagateResult applySchemaChange(final ConnectionAutoPropagateSchemaChange request) throws JsonValidationException, ConfigNotFoundException, IOException { LOGGER.info("Applying schema change for connection '{}' only", request.getConnectionId()); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/OrganizationsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/OrganizationsHandler.java index 0bfede0ac54..7b7511e7bfc 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/OrganizationsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/OrganizationsHandler.java @@ -108,6 +108,11 @@ public OrganizationRead updateOrganization(final OrganizationUpdateRequestBody o organization.setOrgLevelBilling(organizationUpdateRequestBody.getOrgLevelBilling()); hasChanged = true; } + if (organizationUpdateRequestBody.getEmail() != null && !organizationUpdateRequestBody.getEmail() + .equals(organization.getEmail())) { + organization.setEmail(organizationUpdateRequestBody.getEmail()); + hasChanged = true; + } if (hasChanged) { organizationPersistence.updateOrganization(organization); } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/PermissionHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/PermissionHandler.java index eb63b6cd768..6d480df5b81 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/PermissionHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/PermissionHandler.java @@ -20,7 +20,6 @@ import io.airbyte.commons.server.errors.OperationNotAllowedException; import io.airbyte.config.ConfigSchema; import io.airbyte.config.Permission; -import io.airbyte.config.UserPermission; import io.airbyte.config.helpers.PermissionHelper; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.PermissionPersistence; @@ -353,16 +352,8 @@ public PermissionCheckRead permissionsCheckMultipleWorkspaces(final PermissionsC : new PermissionCheckRead().status(StatusEnum.FAILED); } - /** - * Check and get instance_admin permission for a user. - * - * @param userId user id - * @return UserPermission User details with instance_admin permission, null if user does not have - * instance_admin role. - * @throws IOException if there is an issue while interacting with the db. - */ - public UserPermission getUserInstanceAdminPermission(final UUID userId) throws IOException { - return permissionPersistence.getUserInstanceAdminPermission(userId); + public Boolean isUserInstanceAdmin(final UUID userId) throws IOException { + return permissionPersistence.isUserInstanceAdmin(userId); } /** diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java index 49d90cde6c5..b054bd2e804 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java @@ -492,7 +492,7 @@ public void notifySchemaPropagated(final NotificationSettings notificationSettin connectionId, source.getName(), result.changeDescription(), - item.getSlackConfiguration().getWebhook(), + null, List.of(email), isBreakingChange); } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java index 4fea6c20f0e..af0b21e8a96 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java @@ -39,7 +39,6 @@ import io.airbyte.commons.server.errors.ValueConflictKnownException; import io.airbyte.config.Organization; import io.airbyte.config.StandardWorkspace; -import io.airbyte.config.UserPermission; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.ConfigRepository.ResourcesByOrganizationQueryPaginated; @@ -386,8 +385,7 @@ private WorkspaceReadList listWorkspacesByInstanceAdminUser(final ListWorkspaces public WorkspaceReadList listWorkspacesByUser(final ListWorkspacesByUserRequestBody request) throws IOException { // If user has instance_admin permission, list all workspaces. - final UserPermission userInstanceAdminPermission = permissionPersistence.getUserInstanceAdminPermission(request.getUserId()); - if (userInstanceAdminPermission != null) { + if (permissionPersistence.isUserInstanceAdmin(request.getUserId())) { return listWorkspacesByInstanceAdminUser(request); } // User has no instance_admin permission. diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java index 70687c4abb4..bf186b7fbdb 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java @@ -15,9 +15,7 @@ import io.airbyte.api.model.generated.StreamDescriptor; import io.airbyte.api.model.generated.StreamTransform; import io.airbyte.commons.json.Jsons; -import io.airbyte.featureflag.AutoPropagateNewStreams; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.Workspace; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -121,15 +119,13 @@ public static UpdateSchemaResult getUpdatedSchema(final AirbyteCatalog oldCatalo case ADD_STREAM -> { if (nonBreakingChangesPreference.equals(NonBreakingChangesPreference.PROPAGATE_FULLY)) { final var streamAndConfigurationToAdd = newCatalogPerStream.get(streamDescriptor); - if (featureFlagClient.boolVariation(AutoPropagateNewStreams.INSTANCE, new Workspace(workspaceId))) { - // If we're propagating it, we want to enable it! Otherwise, it'll just get dropped when we update - // the catalog. - streamAndConfigurationToAdd.getConfig() - .selected(true); - CatalogConverter.configureDefaultSyncModesForNewStream(streamAndConfigurationToAdd.getStream(), - streamAndConfigurationToAdd.getConfig()); - CatalogConverter.ensureCompatibleDestinationSyncMode(streamAndConfigurationToAdd, supportedDestinationSyncModes); - } + // If we're propagating it, we want to enable it! Otherwise, it'll just get dropped when we update + // the catalog. + streamAndConfigurationToAdd.getConfig() + .selected(true); + CatalogConverter.configureDefaultSyncModesForNewStream(streamAndConfigurationToAdd.getStream(), + streamAndConfigurationToAdd.getConfig()); + CatalogConverter.ensureCompatibleDestinationSyncMode(streamAndConfigurationToAdd, supportedDestinationSyncModes); // TODO(mfsiega-airbyte): handle the case where the chosen sync mode isn't actually one of the // supported sync modes. oldCatalogPerStream.put(streamDescriptor, streamAndConfigurationToAdd); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java index 5658b863e9c..a9af400201a 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java @@ -9,6 +9,7 @@ import io.micronaut.core.util.StringUtils; import jakarta.inject.Singleton; import java.util.Optional; +import java.util.Set; import lombok.extern.slf4j.Slf4j; /** @@ -18,6 +19,12 @@ @Slf4j public class AirbyteHttpRequestFieldExtractor { + // For some APIs we asked for a list of ids, such as workspace IDs and connection IDs. We will + // validate if user has permission + // to all of them. + private static final Set ARRAY_FIELDS = + Set.of(AuthenticationFields.WORKSPACE_IDS_FIELD_NAME, AuthenticationFields.CONNECTION_IDS_FIELD_NAME); + /** * Extracts the requested ID from the HTTP request, if present. * @@ -48,7 +55,7 @@ public Optional extractId(final String content, final String idFieldName } private Optional extract(JsonNode jsonNode, String idFieldName) { - if (idFieldName.equals(AuthenticationFields.WORKSPACE_IDS_FIELD_NAME)) { + if (ARRAY_FIELDS.contains(idFieldName)) { log.debug("Try to extract list of ids for field {}", idFieldName); return Optional.ofNullable(jsonNode.get(idFieldName)) .map(Jsons::serialize) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java index 751a166a20d..f91abce9e5e 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java @@ -15,6 +15,11 @@ public final class AuthenticationFields { */ public static final String CONNECTION_ID_FIELD_NAME = "connectionId"; + /** + * Name of the field in HTTP request bodies that contains the connection IDs value. + */ + public static final String CONNECTION_IDS_FIELD_NAME = "connectionIds"; + /** * Name of the field in HTTP request bodies that contains the destination ID value. */ diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java index 898ea6e90f9..d2e6d277eb7 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java @@ -5,13 +5,13 @@ package io.airbyte.commons.server.support; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONFIG_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_IDS_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.DESTINATION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.JOB_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.OPERATION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.ORGANIZATION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.PERMISSION_ID_HEADER; -import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_DEFINITION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_IDS_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_ID_HEADER; @@ -101,6 +101,8 @@ public List resolveWorkspace(final Map properties) { } else if (properties.containsKey(CONNECTION_ID_HEADER)) { final String connectionId = properties.get(CONNECTION_ID_HEADER); return List.of(workspaceHelper.getWorkspaceForConnectionId(UUID.fromString(connectionId))); + } else if (properties.containsKey(CONNECTION_IDS_HEADER)) { + return resolveConnectionIds(properties); } else if (properties.containsKey(SOURCE_ID_HEADER) && properties.containsKey(DESTINATION_ID_HEADER)) { final String destinationId = properties.get(DESTINATION_ID_HEADER); final String sourceId = properties.get(SOURCE_ID_HEADER); @@ -114,9 +116,6 @@ public List resolveWorkspace(final Map properties) { } else if (properties.containsKey(SOURCE_ID_HEADER)) { final String sourceId = properties.get(SOURCE_ID_HEADER); return List.of(workspaceHelper.getWorkspaceForSourceId(UUID.fromString(sourceId))); - } else if (properties.containsKey(SOURCE_DEFINITION_ID_HEADER)) { - final String sourceDefinitionId = properties.get(SOURCE_DEFINITION_ID_HEADER); - return List.of(workspaceHelper.getWorkspaceForSourceId(UUID.fromString(sourceDefinitionId))); } else if (properties.containsKey(OPERATION_ID_HEADER)) { final String operationId = properties.get(OPERATION_ID_HEADER); return List.of(workspaceHelper.getWorkspaceForOperationId(UUID.fromString(operationId))); @@ -173,4 +172,19 @@ private List resolveWorkspaces(final Map properties) { return null; } + private List resolveConnectionIds(final Map properties) { + final String connectionIds = properties.get(CONNECTION_IDS_HEADER); + if (connectionIds != null) { + final List deserialized = Jsons.deserialize(connectionIds, List.class); + return deserialized.stream().map(connectionId -> { + try { + return workspaceHelper.getWorkspaceForConnectionId(UUID.fromString(connectionId)); + } catch (JsonValidationException | ConfigNotFoundException e) { + throw new RuntimeException(e); + } + }).toList(); + } + return null; + } + } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java index ce8e2c8dd0b..4fac793c14e 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java @@ -25,6 +25,7 @@ public final class AuthenticationHttpHeaders { * HTTP header that contains the connection ID for authorization purposes. */ public static final String CONNECTION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Connection-Id"; + public static final String CONNECTION_IDS_HEADER = AIRBYTE_HEADER_PREFIX + "Connection-Ids"; /** * HTTP header that contains the destination ID for authorization purposes. diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java index e0426e04d68..602c9d15e61 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java @@ -6,6 +6,7 @@ import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.AIRBYTE_USER_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONFIG_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_IDS_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CREATOR_USER_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.DESTINATION_ID_HEADER; @@ -28,6 +29,8 @@ public enum AuthenticationId { EXTERNAL_AUTH_ID(AuthenticationFields.EXTERNAL_AUTH_ID_FIELD_NAME, EXTERNAL_AUTH_ID_HEADER), CONNECTION_ID(AuthenticationFields.CONNECTION_ID_FIELD_NAME, CONNECTION_ID_HEADER), + CONNECTION_IDS(AuthenticationFields.CONNECTION_IDS_FIELD_NAME, CONNECTION_IDS_HEADER), + DESTINATION_ID_(AuthenticationFields.DESTINATION_ID_FIELD_NAME, DESTINATION_ID_HEADER), EMAIL(AuthenticationFields.EMAIL_FIELD_NAME, EMAIL_HEADER), JOB_ID(AuthenticationFields.JOB_ID_FIELD_NAME, JOB_ID_HEADER), diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java index 37ecc7072ef..845946e9846 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java @@ -11,10 +11,12 @@ import static io.airbyte.persistence.job.models.Job.REPLICATION_TYPES; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -31,6 +33,8 @@ import io.airbyte.api.model.generated.ConnectionAutoPropagateResult; import io.airbyte.api.model.generated.ConnectionAutoPropagateSchemaChange; import io.airbyte.api.model.generated.ConnectionCreate; +import io.airbyte.api.model.generated.ConnectionDataHistoryReadItem; +import io.airbyte.api.model.generated.ConnectionDataHistoryRequestBody; import io.airbyte.api.model.generated.ConnectionRead; import io.airbyte.api.model.generated.ConnectionReadList; import io.airbyte.api.model.generated.ConnectionSchedule; @@ -78,6 +82,9 @@ import io.airbyte.config.FieldSelectionData; import io.airbyte.config.Geography; import io.airbyte.config.JobConfig; +import io.airbyte.config.JobConfig.ConfigType; +import io.airbyte.config.JobOutput; +import io.airbyte.config.JobOutput.OutputType; import io.airbyte.config.JobSyncConfig; import io.airbyte.config.Schedule; import io.airbyte.config.Schedule.TimeUnit; @@ -88,7 +95,10 @@ import io.airbyte.config.StandardSync; import io.airbyte.config.StandardSync.ScheduleType; import io.airbyte.config.StandardSync.Status; +import io.airbyte.config.StandardSyncOutput; +import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SyncStats; import io.airbyte.config.persistence.ActorDefinitionVersionHelper; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -98,6 +108,7 @@ import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.persistence.job.models.Attempt; import io.airbyte.persistence.job.models.AttemptStatus; +import io.airbyte.persistence.job.models.AttemptWithJobInfo; import io.airbyte.persistence.job.models.Job; import io.airbyte.persistence.job.models.JobStatus; import io.airbyte.persistence.job.models.JobWithStatusAndTimestamp; @@ -110,8 +121,11 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -125,22 +139,45 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatchers; import org.mockito.Mockito; class ConnectionsHandlerTest { + private static final Instant CURRENT_INSTANT = Instant.now(); + private static final JobWithStatusAndTimestamp FAILED_JOB = + new JobWithStatusAndTimestamp(1, JobStatus.FAILED, CURRENT_INSTANT.getEpochSecond(), CURRENT_INSTANT.getEpochSecond()); + private static final JobWithStatusAndTimestamp SUCCEEDED_JOB = + new JobWithStatusAndTimestamp(1, JobStatus.SUCCEEDED, CURRENT_INSTANT.getEpochSecond(), CURRENT_INSTANT.getEpochSecond()); + private static final JobWithStatusAndTimestamp CANCELLED_JOB = + new JobWithStatusAndTimestamp(1, JobStatus.CANCELLED, CURRENT_INSTANT.getEpochSecond(), CURRENT_INSTANT.getEpochSecond()); + private static final int MAX_FAILURE_JOBS_IN_A_ROW = DEFAULT_FAILED_JOBS_IN_A_ROW_BEFORE_CONNECTION_DISABLE; + private static final int MAX_DAYS_OF_ONLY_FAILED_JOBS = DEFAULT_DAYS_OF_ONLY_FAILED_JOBS_BEFORE_CONNECTION_DISABLE; + private static final int MAX_DAYS_OF_ONLY_FAILED_JOBS_BEFORE_WARNING = DEFAULT_DAYS_OF_ONLY_FAILED_JOBS_BEFORE_CONNECTION_DISABLE / 2; + private static final String PRESTO_TO_HUDI = "presto to hudi"; + private static final String PRESTO_TO_HUDI_PREFIX = "presto_to_hudi"; + private static final String SOURCE_TEST = "source-test"; + private static final String DESTINATION_TEST = "destination-test"; + private static final String CURSOR1 = "cursor1"; + private static final String CURSOR2 = "cursor2"; + private static final String PK1 = "pk1"; + private static final String PK2 = "pk2"; + private static final String PK3 = "pk3"; + private static final String STREAM1 = "stream1"; + private static final String STREAM2 = "stream2"; + private static final String AZKABAN_USERS = "azkaban_users"; + private static final String CRON_TIMEZONE_UTC = "UTC"; + private static final String TIMEZONE_LOS_ANGELES = "America/Los_Angeles"; + private static final String CRON_EXPRESSION = "* */2 * * * ?"; + private static final String STREAM_SELECTION_DATA = "null/users-data0"; private JobPersistence jobPersistence; private ConfigRepository configRepository; private Supplier uuidGenerator; - private ConnectionsHandler connectionsHandler; private UUID workspaceId; private UUID sourceId; private UUID destinationId; private UUID sourceDefinitionId; private UUID destinationDefinitionId; - private SourceConnection source; private DestinationConnection destination; private StandardSync standardSync; @@ -160,33 +197,6 @@ class ConnectionsHandlerTest { private JobNotifier jobNotifier; private Job job; - private static final Instant CURRENT_INSTANT = Instant.now(); - private static final JobWithStatusAndTimestamp FAILED_JOB = - new JobWithStatusAndTimestamp(1, JobStatus.FAILED, CURRENT_INSTANT.getEpochSecond(), CURRENT_INSTANT.getEpochSecond()); - private static final JobWithStatusAndTimestamp SUCCEEDED_JOB = - new JobWithStatusAndTimestamp(1, JobStatus.SUCCEEDED, CURRENT_INSTANT.getEpochSecond(), CURRENT_INSTANT.getEpochSecond()); - private static final JobWithStatusAndTimestamp CANCELLED_JOB = - new JobWithStatusAndTimestamp(1, JobStatus.CANCELLED, CURRENT_INSTANT.getEpochSecond(), CURRENT_INSTANT.getEpochSecond()); - private static final int MAX_FAILURE_JOBS_IN_A_ROW = DEFAULT_FAILED_JOBS_IN_A_ROW_BEFORE_CONNECTION_DISABLE; - private static final int MAX_DAYS_OF_ONLY_FAILED_JOBS = DEFAULT_DAYS_OF_ONLY_FAILED_JOBS_BEFORE_CONNECTION_DISABLE; - private static final int MAX_DAYS_OF_ONLY_FAILED_JOBS_BEFORE_WARNING = DEFAULT_DAYS_OF_ONLY_FAILED_JOBS_BEFORE_CONNECTION_DISABLE / 2; - - private static final String PRESTO_TO_HUDI = "presto to hudi"; - private static final String PRESTO_TO_HUDI_PREFIX = "presto_to_hudi"; - private static final String SOURCE_TEST = "source-test"; - private static final String DESTINATION_TEST = "destination-test"; - private static final String CURSOR1 = "cursor1"; - private static final String CURSOR2 = "cursor2"; - private static final String PK1 = "pk1"; - private static final String PK2 = "pk2"; - private static final String PK3 = "pk3"; - private static final String STREAM1 = "stream1"; - private static final String STREAM2 = "stream2"; - private static final String AZKABAN_USERS = "azkaban_users"; - private static final String CRON_TIMEZONE_UTC = "UTC"; - private static final String CRON_EXPRESSION = "* */2 * * * ?"; - private static final String STREAM_SELECTION_DATA = "null/users-data0"; - @SuppressWarnings("unchecked") @BeforeEach void setUp() throws IOException, JsonValidationException, ConfigNotFoundException { @@ -315,17 +325,290 @@ void setUp() throws JsonValidationException, ConfigNotFoundException, IOExceptio .withName(DESTINATION_TEST) .withDestinationDefinitionId(UUID.randomUUID()); when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); - when(configRepository.getSourceDefinitionFromConnection(standardSync.getConnectionId())).thenReturn( - sourceDefinition); - when(configRepository.getDestinationDefinitionFromConnection(standardSync.getConnectionId())).thenReturn( - destinationDefinition); - when(configRepository.getSourceConnection(source.getSourceId())) - .thenReturn(source); - when(configRepository.getDestinationConnection(destination.getDestinationId())) - .thenReturn(destination); - when(configRepository.getStandardSync(connectionId)).thenReturn(standardSync); - when(jobPersistence.getLastReplicationJob(connectionId)).thenReturn(Optional.of(job)); - when(jobPersistence.getFirstReplicationJob(connectionId)).thenReturn(Optional.of(job)); + when(configRepository.getSourceDefinitionFromConnection(standardSync.getConnectionId())).thenReturn( + sourceDefinition); + when(configRepository.getDestinationDefinitionFromConnection(standardSync.getConnectionId())).thenReturn( + destinationDefinition); + when(configRepository.getSourceConnection(source.getSourceId())) + .thenReturn(source); + when(configRepository.getDestinationConnection(destination.getDestinationId())) + .thenReturn(destination); + when(configRepository.getStandardSync(connectionId)).thenReturn(standardSync); + when(jobPersistence.getLastReplicationJob(connectionId)).thenReturn(Optional.of(job)); + when(jobPersistence.getFirstReplicationJob(connectionId)).thenReturn(Optional.of(job)); + } + + @Test + void testGetConnection() throws JsonValidationException, ConfigNotFoundException, IOException { + when(configRepository.getStandardSync(standardSync.getConnectionId())) + .thenReturn(standardSync); + + final ConnectionRead actualConnectionRead = connectionsHandler.getConnection(standardSync.getConnectionId()); + + assertEquals(ConnectionHelpers.generateExpectedConnectionRead(standardSync), actualConnectionRead); + } + + @Test + void testListConnectionsForWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { + when(configRepository.listWorkspaceStandardSyncs(source.getWorkspaceId(), false)) + .thenReturn(Lists.newArrayList(standardSync)); + when(configRepository.listWorkspaceStandardSyncs(source.getWorkspaceId(), true)) + .thenReturn(Lists.newArrayList(standardSync, standardSyncDeleted)); + when(configRepository.getStandardSync(standardSync.getConnectionId())) + .thenReturn(standardSync); + + final WorkspaceIdRequestBody workspaceIdRequestBody = new WorkspaceIdRequestBody().workspaceId(source.getWorkspaceId()); + final ConnectionReadList actualConnectionReadList = connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody); + assertEquals(1, actualConnectionReadList.getConnections().size()); + assertEquals( + ConnectionHelpers.generateExpectedConnectionRead(standardSync), + actualConnectionReadList.getConnections().get(0)); + + final ConnectionReadList actualConnectionReadListWithDeleted = connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody, true); + final List connections = actualConnectionReadListWithDeleted.getConnections(); + assertEquals(2, connections.size()); + assertEquals(ApiPojoConverters.internalToConnectionRead(standardSync), connections.get(0)); + assertEquals(ApiPojoConverters.internalToConnectionRead(standardSyncDeleted), connections.get(1)); + + } + + @Test + void testListConnections() throws JsonValidationException, ConfigNotFoundException, IOException { + when(configRepository.listStandardSyncs()) + .thenReturn(Lists.newArrayList(standardSync)); + when(configRepository.getSourceConnection(source.getSourceId())) + .thenReturn(source); + when(configRepository.getStandardSync(standardSync.getConnectionId())) + .thenReturn(standardSync); + + final ConnectionReadList actualConnectionReadList = connectionsHandler.listConnections(); + + assertEquals( + ConnectionHelpers.generateExpectedConnectionRead(standardSync), + actualConnectionReadList.getConnections().get(0)); + } + + @Test + void testListConnectionsByActorDefinition() throws IOException { + when(configRepository.listConnectionsByActorDefinitionIdAndType(sourceDefinitionId, ActorType.SOURCE.value(), false)) + .thenReturn(Lists.newArrayList(standardSync)); + when(configRepository.listConnectionsByActorDefinitionIdAndType(destinationDefinitionId, ActorType.DESTINATION.value(), false)) + .thenReturn(Lists.newArrayList(standardSync2)); + + final ConnectionReadList connectionReadListForSourceDefinitionId = connectionsHandler.listConnectionsForActorDefinition( + new ActorDefinitionRequestBody() + .actorDefinitionId(sourceDefinitionId) + .actorType(io.airbyte.api.model.generated.ActorType.SOURCE)); + + final ConnectionReadList connectionReadListForDestinationDefinitionId = connectionsHandler.listConnectionsForActorDefinition( + new ActorDefinitionRequestBody() + .actorDefinitionId(destinationDefinitionId) + .actorType(io.airbyte.api.model.generated.ActorType.DESTINATION)); + + assertEquals( + List.of(ConnectionHelpers.generateExpectedConnectionRead(standardSync)), + connectionReadListForSourceDefinitionId.getConnections()); + assertEquals( + List.of(ConnectionHelpers.generateExpectedConnectionRead(standardSync2)), + connectionReadListForDestinationDefinitionId.getConnections()); + } + + @Test + void testSearchConnections() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConnectionRead connectionRead1 = ConnectionHelpers.connectionReadFromStandardSync(standardSync); + final StandardSync standardSync2 = new StandardSync() + .withConnectionId(UUID.randomUUID()) + .withName("test connection") + .withNamespaceDefinition(JobSyncConfig.NamespaceDefinitionType.CUSTOMFORMAT) + .withNamespaceFormat("ns_format") + .withPrefix("test_prefix") + .withStatus(StandardSync.Status.ACTIVE) + .withCatalog(ConnectionHelpers.generateBasicConfiguredAirbyteCatalog()) + .withSourceId(sourceId) + .withDestinationId(destinationId) + .withOperationIds(List.of(operationId)) + .withManual(true) + .withResourceRequirements(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS) + .withGeography(Geography.US) + .withBreakingChange(false) + .withNotifySchemaChanges(false) + .withNotifySchemaChangesByEmail(true); + final ConnectionRead connectionRead2 = ConnectionHelpers.connectionReadFromStandardSync(standardSync2); + final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() + .withName(SOURCE_TEST) + .withSourceDefinitionId(UUID.randomUUID()); + final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() + .withName(DESTINATION_TEST) + .withDestinationDefinitionId(UUID.randomUUID()); + + when(configRepository.listStandardSyncs()) + .thenReturn(Lists.newArrayList(standardSync, standardSync2)); + when(configRepository.getSourceConnection(source.getSourceId())) + .thenReturn(source); + when(configRepository.getDestinationConnection(destination.getDestinationId())) + .thenReturn(destination); + when(configRepository.getStandardSync(standardSync.getConnectionId())) + .thenReturn(standardSync); + when(configRepository.getStandardSync(standardSync2.getConnectionId())) + .thenReturn(standardSync2); + when(configRepository.getStandardSourceDefinition(source.getSourceDefinitionId())) + .thenReturn(sourceDefinition); + when(configRepository.getStandardDestinationDefinition(destination.getDestinationDefinitionId())) + .thenReturn(destinationDefinition); + + final ConnectionSearch connectionSearch = new ConnectionSearch(); + connectionSearch.namespaceDefinition(NamespaceDefinitionType.SOURCE); + ConnectionReadList actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(1, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + + connectionSearch.namespaceDefinition(null); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(2, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + + final SourceSearch sourceSearch = new SourceSearch().sourceId(UUID.randomUUID()); + connectionSearch.setSource(sourceSearch); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(0, actualConnectionReadList.getConnections().size()); + + sourceSearch.sourceId(connectionRead1.getSourceId()); + connectionSearch.setSource(sourceSearch); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(2, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + + final DestinationSearch destinationSearch = new DestinationSearch(); + connectionSearch.setDestination(destinationSearch); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(2, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + + destinationSearch.connectionConfiguration(Jsons.jsonNode(Collections.singletonMap("apiKey", "not-found"))); + connectionSearch.setDestination(destinationSearch); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(0, actualConnectionReadList.getConnections().size()); + + destinationSearch.connectionConfiguration(Jsons.jsonNode(Collections.singletonMap("apiKey", "123-abc"))); + connectionSearch.setDestination(destinationSearch); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(2, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + + connectionSearch.name("non-existent"); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(0, actualConnectionReadList.getConnections().size()); + + connectionSearch.name(connectionRead1.getName()); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(1, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + + connectionSearch.name(connectionRead2.getName()); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(1, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(0)); + + connectionSearch.namespaceDefinition(connectionRead1.getNamespaceDefinition()); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(0, actualConnectionReadList.getConnections().size()); + + connectionSearch.name(null); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(1, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + + connectionSearch.namespaceDefinition(connectionRead2.getNamespaceDefinition()); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(1, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(0)); + + connectionSearch.namespaceDefinition(null); + connectionSearch.status(ConnectionStatus.INACTIVE); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(0, actualConnectionReadList.getConnections().size()); + + connectionSearch.status(ConnectionStatus.ACTIVE); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(2, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + + connectionSearch.prefix(connectionRead1.getPrefix()); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(1, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + + connectionSearch.prefix(connectionRead2.getPrefix()); + actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); + assertEquals(1, actualConnectionReadList.getConnections().size()); + assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(0)); + } + + @Test + void testDeleteConnection() throws JsonValidationException, ConfigNotFoundException, IOException { + connectionsHandler.deleteConnection(connectionId); + + verify(connectionHelper).deleteConnection(connectionId); + } + + @Test + void failOnUnmatchedWorkspacesInCreate() throws JsonValidationException, ConfigNotFoundException, IOException { + when(workspaceHelper.getWorkspaceForSourceIdIgnoreExceptions(standardSync.getSourceId())).thenReturn(UUID.randomUUID()); + when(workspaceHelper.getWorkspaceForDestinationIdIgnoreExceptions(standardSync.getDestinationId())).thenReturn(UUID.randomUUID()); + when(configRepository.getSourceConnection(source.getSourceId())) + .thenReturn(source); + when(configRepository.getDestinationConnection(destination.getDestinationId())) + .thenReturn(destination); + + when(uuidGenerator.get()).thenReturn(standardSync.getConnectionId()); + final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() + .withName(SOURCE_TEST) + .withSourceDefinitionId(UUID.randomUUID()); + final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() + .withName(DESTINATION_TEST) + .withDestinationDefinitionId(UUID.randomUUID()); + when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); + when(configRepository.getSourceDefinitionFromConnection(standardSync.getConnectionId())).thenReturn(sourceDefinition); + when(configRepository.getDestinationDefinitionFromConnection(standardSync.getConnectionId())).thenReturn(destinationDefinition); + + final AirbyteCatalog catalog = ConnectionHelpers.generateBasicApiCatalog(); + + final ConnectionCreate connectionCreate = new ConnectionCreate() + .sourceId(standardSync.getSourceId()) + .destinationId(standardSync.getDestinationId()) + .operationIds(standardSync.getOperationIds()) + .name(PRESTO_TO_HUDI) + .namespaceDefinition(NamespaceDefinitionType.SOURCE) + .namespaceFormat(null) + .prefix(PRESTO_TO_HUDI_PREFIX) + .status(ConnectionStatus.ACTIVE) + .schedule(ConnectionHelpers.generateBasicConnectionSchedule()) + .syncCatalog(catalog) + .resourceRequirements(new io.airbyte.api.model.generated.ResourceRequirements() + .cpuRequest(standardSync.getResourceRequirements().getCpuRequest()) + .cpuLimit(standardSync.getResourceRequirements().getCpuLimit()) + .memoryRequest(standardSync.getResourceRequirements().getMemoryRequest()) + .memoryLimit(standardSync.getResourceRequirements().getMemoryLimit())); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + connectionsHandler.createConnection(connectionCreate); + }); + } + + @Test + void testEnumConversion() { + assertTrue(Enums.isCompatible(ConnectionStatus.class, StandardSync.Status.class)); + assertTrue(Enums.isCompatible(io.airbyte.config.SyncMode.class, SyncMode.class)); + assertTrue(Enums.isCompatible(StandardSync.Status.class, ConnectionStatus.class)); + assertTrue(Enums.isCompatible(ConnectionSchedule.TimeUnitEnum.class, Schedule.TimeUnit.class)); + assertTrue(Enums.isCompatible(io.airbyte.api.model.generated.DataType.class, DataType.class)); + assertTrue(Enums.isCompatible(DataType.class, io.airbyte.api.model.generated.DataType.class)); + assertTrue(Enums.isCompatible(NamespaceDefinitionType.class, io.airbyte.config.JobSyncConfig.NamespaceDefinitionType.class)); } @Nested @@ -528,7 +811,7 @@ private void verifyDisabled() throws IOException { argThat(standardSync -> (standardSync.getStatus().equals(Status.INACTIVE) && standardSync.getConnectionId().equals(connectionId)))); verify(configRepository, times(1)).writeStandardSync(standardSync); verify(jobNotifier, times(1)).autoDisableConnection(job); - verify(jobNotifier, times(1)).notifyJobByEmail(any(), any(), ArgumentMatchers.eq(job)); + verify(jobNotifier, times(1)).notifyJobByEmail(any(), any(), eq(job)); verify(jobNotifier, Mockito.never()).autoDisableConnectionWarning(any()); } @@ -1016,380 +1299,226 @@ void testUpdateConnectionPatchColumnSelection() throws Exception { final ConnectionRead expectedRead = ConnectionHelpers.generateExpectedConnectionRead(standardSync) .syncCatalog(catalogForUpdate); - final StandardSync expectedPersistedSync = Jsons.clone(standardSync) - .withCatalog(expectedPersistedCatalog) - .withFieldSelectionData(CatalogConverter.getFieldSelectionData(catalogForUpdate)); - - when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); - - final ConnectionRead actualConnectionRead = connectionsHandler.updateConnection(connectionUpdate); - - assertEquals(expectedRead, actualConnectionRead); - verify(configRepository).writeStandardSync(expectedPersistedSync); - verify(eventRunner).update(connectionUpdate.getConnectionId()); - } - - @Test - void testUpdateConnectionPatchingSeveralFieldsAndReplaceAStream() throws JsonValidationException, ConfigNotFoundException, IOException { - final AirbyteCatalog catalogForUpdate = ConnectionHelpers.generateMultipleStreamsApiCatalog(2); - - // deselect the existing stream, and add a new stream called 'azkaban_users'. - // result that we persist and read after update should be a catalog with a single - // stream called 'azkaban_users'. - catalogForUpdate.getStreams().get(0).getConfig().setSelected(false); - catalogForUpdate.getStreams().get(1).getStream().setName(AZKABAN_USERS); - catalogForUpdate.getStreams().get(1).getConfig().setAliasName(AZKABAN_USERS); - - final UUID newSourceCatalogId = UUID.randomUUID(); - - final ResourceRequirements resourceRequirements = new ResourceRequirements() - .cpuLimit(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS.getCpuLimit()) - .cpuRequest(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS.getCpuRequest()) - .memoryLimit(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS.getMemoryLimit()) - .memoryRequest(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS.getMemoryRequest()); - - final ConnectionUpdate connectionUpdate = new ConnectionUpdate() - .connectionId(standardSync.getConnectionId()) - .status(ConnectionStatus.INACTIVE) - .scheduleType(ConnectionScheduleType.MANUAL) - .syncCatalog(catalogForUpdate) - .resourceRequirements(resourceRequirements) - .sourceCatalogId(newSourceCatalogId) - .operationIds(List.of(operationId, otherOperationId)) - .geography(io.airbyte.api.model.generated.Geography.EU); - - final ConfiguredAirbyteCatalog expectedPersistedCatalog = ConnectionHelpers.generateBasicConfiguredAirbyteCatalog(); - expectedPersistedCatalog.getStreams().get(0).getStream().withName(AZKABAN_USERS); - - final StandardSync expectedPersistedSync = Jsons.clone(standardSync) - .withStatus(Status.INACTIVE) - .withScheduleType(ScheduleType.MANUAL) - .withScheduleData(null) - .withSchedule(null) - .withManual(true) - .withCatalog(expectedPersistedCatalog) - .withFieldSelectionData(CatalogConverter.getFieldSelectionData(catalogForUpdate)) - .withResourceRequirements(ApiPojoConverters.resourceRequirementsToInternal(resourceRequirements)) - .withSourceCatalogId(newSourceCatalogId) - .withOperationIds(List.of(operationId, otherOperationId)) - .withGeography(Geography.EU); - - when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); - - final ConnectionRead actualConnectionRead = connectionsHandler.updateConnection(connectionUpdate); - - final AirbyteCatalog expectedCatalogInRead = ConnectionHelpers.generateBasicApiCatalog(); - expectedCatalogInRead.getStreams().get(0).getStream().setName(AZKABAN_USERS); - expectedCatalogInRead.getStreams().get(0).getConfig().setAliasName(AZKABAN_USERS); - - final ConnectionRead expectedConnectionRead = ConnectionHelpers.generateExpectedConnectionRead( - standardSync.getConnectionId(), - standardSync.getSourceId(), - standardSync.getDestinationId(), - standardSync.getOperationIds(), - newSourceCatalogId, - ApiPojoConverters.toApiGeography(standardSync.getGeography()), - false, - standardSync.getNotifySchemaChanges(), - standardSync.getNotifySchemaChangesByEmail()) - .status(ConnectionStatus.INACTIVE) - .scheduleType(ConnectionScheduleType.MANUAL) - .scheduleData(null) - .schedule(null) - .syncCatalog(expectedCatalogInRead) - .resourceRequirements(resourceRequirements); - - assertEquals(expectedConnectionRead, actualConnectionRead); - verify(configRepository).writeStandardSync(expectedPersistedSync); - verify(eventRunner).update(connectionUpdate.getConnectionId()); - } - - @Test - void testValidateConnectionUpdateOperationInDifferentWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { - when(workspaceHelper.getWorkspaceForOperationIdIgnoreExceptions(operationId)).thenReturn(UUID.randomUUID()); - when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); - - final ConnectionUpdate connectionUpdate = new ConnectionUpdate() - .connectionId(standardSync.getConnectionId()) - .operationIds(Collections.singletonList(operationId)) - .syncCatalog(CatalogConverter.toApi(standardSync.getCatalog(), standardSync.getFieldSelectionData())); - - assertThrows(IllegalArgumentException.class, () -> connectionsHandler.updateConnection(connectionUpdate)); - } - - } - - @Test - void testGetConnection() throws JsonValidationException, ConfigNotFoundException, IOException { - when(configRepository.getStandardSync(standardSync.getConnectionId())) - .thenReturn(standardSync); - - final ConnectionRead actualConnectionRead = connectionsHandler.getConnection(standardSync.getConnectionId()); - - assertEquals(ConnectionHelpers.generateExpectedConnectionRead(standardSync), actualConnectionRead); - } - - @Test - void testListConnectionsForWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { - when(configRepository.listWorkspaceStandardSyncs(source.getWorkspaceId(), false)) - .thenReturn(Lists.newArrayList(standardSync)); - when(configRepository.listWorkspaceStandardSyncs(source.getWorkspaceId(), true)) - .thenReturn(Lists.newArrayList(standardSync, standardSyncDeleted)); - when(configRepository.getStandardSync(standardSync.getConnectionId())) - .thenReturn(standardSync); - - final WorkspaceIdRequestBody workspaceIdRequestBody = new WorkspaceIdRequestBody().workspaceId(source.getWorkspaceId()); - final ConnectionReadList actualConnectionReadList = connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody); - assertEquals(1, actualConnectionReadList.getConnections().size()); - assertEquals( - ConnectionHelpers.generateExpectedConnectionRead(standardSync), - actualConnectionReadList.getConnections().get(0)); - - final ConnectionReadList actualConnectionReadListWithDeleted = connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody, true); - final List connections = actualConnectionReadListWithDeleted.getConnections(); - assertEquals(2, connections.size()); - assertEquals(ApiPojoConverters.internalToConnectionRead(standardSync), connections.get(0)); - assertEquals(ApiPojoConverters.internalToConnectionRead(standardSyncDeleted), connections.get(1)); - - } - - @Test - void testListConnections() throws JsonValidationException, ConfigNotFoundException, IOException { - when(configRepository.listStandardSyncs()) - .thenReturn(Lists.newArrayList(standardSync)); - when(configRepository.getSourceConnection(source.getSourceId())) - .thenReturn(source); - when(configRepository.getStandardSync(standardSync.getConnectionId())) - .thenReturn(standardSync); - - final ConnectionReadList actualConnectionReadList = connectionsHandler.listConnections(); - - assertEquals( - ConnectionHelpers.generateExpectedConnectionRead(standardSync), - actualConnectionReadList.getConnections().get(0)); - } - - @Test - void testListConnectionsByActorDefinition() throws IOException { - when(configRepository.listConnectionsByActorDefinitionIdAndType(sourceDefinitionId, ActorType.SOURCE.value(), false)) - .thenReturn(Lists.newArrayList(standardSync)); - when(configRepository.listConnectionsByActorDefinitionIdAndType(destinationDefinitionId, ActorType.DESTINATION.value(), false)) - .thenReturn(Lists.newArrayList(standardSync2)); - - final ConnectionReadList connectionReadListForSourceDefinitionId = connectionsHandler.listConnectionsForActorDefinition( - new ActorDefinitionRequestBody() - .actorDefinitionId(sourceDefinitionId) - .actorType(io.airbyte.api.model.generated.ActorType.SOURCE)); - - final ConnectionReadList connectionReadListForDestinationDefinitionId = connectionsHandler.listConnectionsForActorDefinition( - new ActorDefinitionRequestBody() - .actorDefinitionId(destinationDefinitionId) - .actorType(io.airbyte.api.model.generated.ActorType.DESTINATION)); - - assertEquals( - List.of(ConnectionHelpers.generateExpectedConnectionRead(standardSync)), - connectionReadListForSourceDefinitionId.getConnections()); - assertEquals( - List.of(ConnectionHelpers.generateExpectedConnectionRead(standardSync2)), - connectionReadListForDestinationDefinitionId.getConnections()); - } - - @Test - void testSearchConnections() throws JsonValidationException, ConfigNotFoundException, IOException { - final ConnectionRead connectionRead1 = ConnectionHelpers.connectionReadFromStandardSync(standardSync); - final StandardSync standardSync2 = new StandardSync() - .withConnectionId(UUID.randomUUID()) - .withName("test connection") - .withNamespaceDefinition(JobSyncConfig.NamespaceDefinitionType.CUSTOMFORMAT) - .withNamespaceFormat("ns_format") - .withPrefix("test_prefix") - .withStatus(StandardSync.Status.ACTIVE) - .withCatalog(ConnectionHelpers.generateBasicConfiguredAirbyteCatalog()) - .withSourceId(sourceId) - .withDestinationId(destinationId) - .withOperationIds(List.of(operationId)) - .withManual(true) - .withResourceRequirements(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS) - .withGeography(Geography.US) - .withBreakingChange(false) - .withNotifySchemaChanges(false) - .withNotifySchemaChangesByEmail(true); - final ConnectionRead connectionRead2 = ConnectionHelpers.connectionReadFromStandardSync(standardSync2); - final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() - .withName(SOURCE_TEST) - .withSourceDefinitionId(UUID.randomUUID()); - final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() - .withName(DESTINATION_TEST) - .withDestinationDefinitionId(UUID.randomUUID()); - - when(configRepository.listStandardSyncs()) - .thenReturn(Lists.newArrayList(standardSync, standardSync2)); - when(configRepository.getSourceConnection(source.getSourceId())) - .thenReturn(source); - when(configRepository.getDestinationConnection(destination.getDestinationId())) - .thenReturn(destination); - when(configRepository.getStandardSync(standardSync.getConnectionId())) - .thenReturn(standardSync); - when(configRepository.getStandardSync(standardSync2.getConnectionId())) - .thenReturn(standardSync2); - when(configRepository.getStandardSourceDefinition(source.getSourceDefinitionId())) - .thenReturn(sourceDefinition); - when(configRepository.getStandardDestinationDefinition(destination.getDestinationDefinitionId())) - .thenReturn(destinationDefinition); + final StandardSync expectedPersistedSync = Jsons.clone(standardSync) + .withCatalog(expectedPersistedCatalog) + .withFieldSelectionData(CatalogConverter.getFieldSelectionData(catalogForUpdate)); - final ConnectionSearch connectionSearch = new ConnectionSearch(); - connectionSearch.namespaceDefinition(NamespaceDefinitionType.SOURCE); - ConnectionReadList actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(1, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); - connectionSearch.namespaceDefinition(null); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(2, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); - assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + final ConnectionRead actualConnectionRead = connectionsHandler.updateConnection(connectionUpdate); - final SourceSearch sourceSearch = new SourceSearch().sourceId(UUID.randomUUID()); - connectionSearch.setSource(sourceSearch); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(0, actualConnectionReadList.getConnections().size()); + assertEquals(expectedRead, actualConnectionRead); + verify(configRepository).writeStandardSync(expectedPersistedSync); + verify(eventRunner).update(connectionUpdate.getConnectionId()); + } - sourceSearch.sourceId(connectionRead1.getSourceId()); - connectionSearch.setSource(sourceSearch); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(2, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); - assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + @Test + void testUpdateConnectionPatchingSeveralFieldsAndReplaceAStream() throws JsonValidationException, ConfigNotFoundException, IOException { + final AirbyteCatalog catalogForUpdate = ConnectionHelpers.generateMultipleStreamsApiCatalog(2); - final DestinationSearch destinationSearch = new DestinationSearch(); - connectionSearch.setDestination(destinationSearch); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(2, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); - assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + // deselect the existing stream, and add a new stream called 'azkaban_users'. + // result that we persist and read after update should be a catalog with a single + // stream called 'azkaban_users'. + catalogForUpdate.getStreams().get(0).getConfig().setSelected(false); + catalogForUpdate.getStreams().get(1).getStream().setName(AZKABAN_USERS); + catalogForUpdate.getStreams().get(1).getConfig().setAliasName(AZKABAN_USERS); - destinationSearch.connectionConfiguration(Jsons.jsonNode(Collections.singletonMap("apiKey", "not-found"))); - connectionSearch.setDestination(destinationSearch); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(0, actualConnectionReadList.getConnections().size()); + final UUID newSourceCatalogId = UUID.randomUUID(); - destinationSearch.connectionConfiguration(Jsons.jsonNode(Collections.singletonMap("apiKey", "123-abc"))); - connectionSearch.setDestination(destinationSearch); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(2, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); - assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + final ResourceRequirements resourceRequirements = new ResourceRequirements() + .cpuLimit(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS.getCpuLimit()) + .cpuRequest(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS.getCpuRequest()) + .memoryLimit(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS.getMemoryLimit()) + .memoryRequest(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS.getMemoryRequest()); - connectionSearch.name("non-existent"); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(0, actualConnectionReadList.getConnections().size()); + final ConnectionUpdate connectionUpdate = new ConnectionUpdate() + .connectionId(standardSync.getConnectionId()) + .status(ConnectionStatus.INACTIVE) + .scheduleType(ConnectionScheduleType.MANUAL) + .syncCatalog(catalogForUpdate) + .resourceRequirements(resourceRequirements) + .sourceCatalogId(newSourceCatalogId) + .operationIds(List.of(operationId, otherOperationId)) + .geography(io.airbyte.api.model.generated.Geography.EU); - connectionSearch.name(connectionRead1.getName()); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(1, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + final ConfiguredAirbyteCatalog expectedPersistedCatalog = ConnectionHelpers.generateBasicConfiguredAirbyteCatalog(); + expectedPersistedCatalog.getStreams().get(0).getStream().withName(AZKABAN_USERS); - connectionSearch.name(connectionRead2.getName()); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(1, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(0)); + final StandardSync expectedPersistedSync = Jsons.clone(standardSync) + .withStatus(Status.INACTIVE) + .withScheduleType(ScheduleType.MANUAL) + .withScheduleData(null) + .withSchedule(null) + .withManual(true) + .withCatalog(expectedPersistedCatalog) + .withFieldSelectionData(CatalogConverter.getFieldSelectionData(catalogForUpdate)) + .withResourceRequirements(ApiPojoConverters.resourceRequirementsToInternal(resourceRequirements)) + .withSourceCatalogId(newSourceCatalogId) + .withOperationIds(List.of(operationId, otherOperationId)) + .withGeography(Geography.EU); - connectionSearch.namespaceDefinition(connectionRead1.getNamespaceDefinition()); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(0, actualConnectionReadList.getConnections().size()); + when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); - connectionSearch.name(null); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(1, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + final ConnectionRead actualConnectionRead = connectionsHandler.updateConnection(connectionUpdate); - connectionSearch.namespaceDefinition(connectionRead2.getNamespaceDefinition()); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(1, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(0)); + final AirbyteCatalog expectedCatalogInRead = ConnectionHelpers.generateBasicApiCatalog(); + expectedCatalogInRead.getStreams().get(0).getStream().setName(AZKABAN_USERS); + expectedCatalogInRead.getStreams().get(0).getConfig().setAliasName(AZKABAN_USERS); - connectionSearch.namespaceDefinition(null); - connectionSearch.status(ConnectionStatus.INACTIVE); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(0, actualConnectionReadList.getConnections().size()); + final ConnectionRead expectedConnectionRead = ConnectionHelpers.generateExpectedConnectionRead( + standardSync.getConnectionId(), + standardSync.getSourceId(), + standardSync.getDestinationId(), + standardSync.getOperationIds(), + newSourceCatalogId, + ApiPojoConverters.toApiGeography(standardSync.getGeography()), + false, + standardSync.getNotifySchemaChanges(), + standardSync.getNotifySchemaChangesByEmail()) + .status(ConnectionStatus.INACTIVE) + .scheduleType(ConnectionScheduleType.MANUAL) + .scheduleData(null) + .schedule(null) + .syncCatalog(expectedCatalogInRead) + .resourceRequirements(resourceRequirements); - connectionSearch.status(ConnectionStatus.ACTIVE); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(2, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); - assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(1)); + assertEquals(expectedConnectionRead, actualConnectionRead); + verify(configRepository).writeStandardSync(expectedPersistedSync); + verify(eventRunner).update(connectionUpdate.getConnectionId()); + } - connectionSearch.prefix(connectionRead1.getPrefix()); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(1, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead1, actualConnectionReadList.getConnections().get(0)); + @Test + void testValidateConnectionUpdateOperationInDifferentWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { + when(workspaceHelper.getWorkspaceForOperationIdIgnoreExceptions(operationId)).thenReturn(UUID.randomUUID()); + when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); - connectionSearch.prefix(connectionRead2.getPrefix()); - actualConnectionReadList = connectionsHandler.searchConnections(connectionSearch); - assertEquals(1, actualConnectionReadList.getConnections().size()); - assertEquals(connectionRead2, actualConnectionReadList.getConnections().get(0)); - } + final ConnectionUpdate connectionUpdate = new ConnectionUpdate() + .connectionId(standardSync.getConnectionId()) + .operationIds(Collections.singletonList(operationId)) + .syncCatalog(CatalogConverter.toApi(standardSync.getCatalog(), standardSync.getFieldSelectionData())); - @Test - void testDeleteConnection() throws JsonValidationException, ConfigNotFoundException, IOException { - connectionsHandler.deleteConnection(connectionId); + assertThrows(IllegalArgumentException.class, () -> connectionsHandler.updateConnection(connectionUpdate)); + } - verify(connectionHelper).deleteConnection(connectionId); } - @Test - void failOnUnmatchedWorkspacesInCreate() throws JsonValidationException, ConfigNotFoundException, IOException { - when(workspaceHelper.getWorkspaceForSourceIdIgnoreExceptions(standardSync.getSourceId())).thenReturn(UUID.randomUUID()); - when(workspaceHelper.getWorkspaceForDestinationIdIgnoreExceptions(standardSync.getDestinationId())).thenReturn(UUID.randomUUID()); - when(configRepository.getSourceConnection(source.getSourceId())) - .thenReturn(source); - when(configRepository.getDestinationConnection(destination.getDestinationId())) - .thenReturn(destination); + } - when(uuidGenerator.get()).thenReturn(standardSync.getConnectionId()); - final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() - .withName(SOURCE_TEST) - .withSourceDefinitionId(UUID.randomUUID()); - final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() - .withName(DESTINATION_TEST) - .withDestinationDefinitionId(UUID.randomUUID()); - when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); - when(configRepository.getSourceDefinitionFromConnection(standardSync.getConnectionId())).thenReturn(sourceDefinition); - when(configRepository.getDestinationDefinitionFromConnection(standardSync.getConnectionId())).thenReturn(destinationDefinition); + @Nested + class ConnectionHistory { - final AirbyteCatalog catalog = ConnectionHelpers.generateBasicApiCatalog(); + @BeforeEach + void setUp() { + // todo: is this unneeded? i think it's already done in the @BeforeAll + connectionsHandler = new ConnectionsHandler( + jobPersistence, + configRepository, + uuidGenerator, + workspaceHelper, + trackingClient, + eventRunner, + connectionHelper, + featureFlagClient, + actorDefinitionVersionHelper, + connectorDefinitionSpecificationHandler, + jobNotifier, + MAX_DAYS_OF_ONLY_FAILED_JOBS, + MAX_FAILURE_JOBS_IN_A_ROW); + } - final ConnectionCreate connectionCreate = new ConnectionCreate() - .sourceId(standardSync.getSourceId()) - .destinationId(standardSync.getDestinationId()) - .operationIds(standardSync.getOperationIds()) - .name(PRESTO_TO_HUDI) - .namespaceDefinition(NamespaceDefinitionType.SOURCE) - .namespaceFormat(null) - .prefix(PRESTO_TO_HUDI_PREFIX) - .status(ConnectionStatus.ACTIVE) - .schedule(ConnectionHelpers.generateBasicConnectionSchedule()) - .syncCatalog(catalog) - .resourceRequirements(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(standardSync.getResourceRequirements().getCpuRequest()) - .cpuLimit(standardSync.getResourceRequirements().getCpuLimit()) - .memoryRequest(standardSync.getResourceRequirements().getMemoryRequest()) - .memoryLimit(standardSync.getResourceRequirements().getMemoryLimit())); + private Attempt generateMockAttempt(final Instant attemptTime, final long bytesSynced) { + final StandardSyncSummary standardSyncSummary = new StandardSyncSummary().withTotalStats(new SyncStats().withBytesCommitted(bytesSynced)); + final StandardSyncOutput standardSyncOutput = new StandardSyncOutput().withStandardSyncSummary(standardSyncSummary); + final JobOutput jobOutput = new JobOutput().withOutputType(OutputType.SYNC).withSync(standardSyncOutput); + return new Attempt(0, 0, null, null, jobOutput, AttemptStatus.FAILED, null, null, 0, 0, attemptTime.getEpochSecond()); + } - Assert.assertThrows(IllegalArgumentException.class, () -> { - connectionsHandler.createConnection(connectionCreate); - }); + private Job generateMockJob(final UUID connectionId, final Attempt attempt) { + return new Job(0L, JobConfig.ConfigType.SYNC, connectionId.toString(), null, List.of(attempt), JobStatus.RUNNING, 1001L, 1000L, 1002L); } - @Test - void testEnumConversion() { - assertTrue(Enums.isCompatible(ConnectionStatus.class, StandardSync.Status.class)); - assertTrue(Enums.isCompatible(io.airbyte.config.SyncMode.class, SyncMode.class)); - assertTrue(Enums.isCompatible(StandardSync.Status.class, ConnectionStatus.class)); - assertTrue(Enums.isCompatible(ConnectionSchedule.TimeUnitEnum.class, Schedule.TimeUnit.class)); - assertTrue(Enums.isCompatible(io.airbyte.api.model.generated.DataType.class, DataType.class)); - assertTrue(Enums.isCompatible(DataType.class, io.airbyte.api.model.generated.DataType.class)); - assertTrue(Enums.isCompatible(NamespaceDefinitionType.class, io.airbyte.config.JobSyncConfig.NamespaceDefinitionType.class)); + private List generateEmptyConnectionDataHistoryReadList(final LocalDate startDate, + final LocalDate endDate, + final String timezone) { + final List connectionDataHistoryReadList = new ArrayList<>(); + for (LocalDate date = startDate; !date.isAfter(endDate); date = date.plusDays(1)) { + connectionDataHistoryReadList.add(new ConnectionDataHistoryReadItem() + .timestamp(Math.toIntExact(date.atStartOfDay(ZoneId.of(timezone)).toEpochSecond())) + .bytes(0)); + } + return connectionDataHistoryReadList; + } + + @Nested + class GetConnectionDataHistory { + + @Test + @DisplayName("Handles empty history response") + void testDataHistoryWithEmptyResponse() throws JsonValidationException, ConfigNotFoundException, IOException { + // test that when getConnectionDataHistory is returned an empty list of attempts, the response + // contains 30 entries all set to 0 bytesCommitted + final ConnectionRead connectionRead = new ConnectionRead() + .connectionId(UUID.randomUUID()) + .syncCatalog(new AirbyteCatalog().streams(Collections.emptyList())); + final ConnectionDataHistoryRequestBody requestBody = new ConnectionDataHistoryRequestBody() + .connectionId(connectionRead.getConnectionId()) + .timezone(TIMEZONE_LOS_ANGELES); + // todo: does this do weird things if this test is run near a day boundary and/or daylight savings + // time? can or should i tell it to mock the actual time to match? + final LocalDate startDate = Instant.now().atZone(ZoneId.of(requestBody.getTimezone())).minusDays(30).toLocalDate(); + final LocalDate endDate = LocalDate.now(ZoneId.of(requestBody.getTimezone())); + + final List actual = connectionsHandler.getConnectionDataHistory(requestBody); + // expected should be a list of items that has 30 entries, all with 0 bytesCommitted and each with a + // timestamp that is 1 day apart + final List expected = generateEmptyConnectionDataHistoryReadList(startDate, endDate, + requestBody.getTimezone()); + + assertEquals(expected, actual); + } + + @Test + @DisplayName("Aggregates data correctly") + void testDataHistoryAggregation() throws JsonValidationException, ConfigNotFoundException, IOException { + final UUID connectionId = UUID.randomUUID(); + final Instant endTime = Instant.now(); + final Instant startTime = endTime.minus(30, ChronoUnit.DAYS); + final long attempt1Bytes = 100L; + final long attempt2Bytes = 150L; + final long attempt3Bytes = 200L; + + // First Attempt - Day 1 + final Attempt attempt1 = generateMockAttempt(startTime.plus(1, ChronoUnit.DAYS), attempt1Bytes); // 100 bytes + final AttemptWithJobInfo attemptWithJobInfo1 = new AttemptWithJobInfo(attempt1, generateMockJob(connectionId, attempt1)); + + // Second Attempt - Same Day as First + final Attempt attempt2 = generateMockAttempt(startTime.plus(1, ChronoUnit.DAYS), attempt2Bytes); // 150 bytes + final AttemptWithJobInfo attemptWithJobInfo2 = new AttemptWithJobInfo(attempt2, generateMockJob(connectionId, attempt2)); + + // Third Attempt - Different Day + final Attempt attempt3 = generateMockAttempt(startTime.plus(2, ChronoUnit.DAYS), attempt3Bytes); // 200 bytes + final AttemptWithJobInfo attemptWithJobInfo3 = new AttemptWithJobInfo(attempt3, generateMockJob(connectionId, attempt3)); + + final List attempts = Arrays.asList(attemptWithJobInfo1, attemptWithJobInfo2, attemptWithJobInfo3); + + when(jobPersistence.listAttemptsForConnectionAfterTimestamp(eq(connectionId), eq(ConfigType.SYNC), any(Instant.class))) + .thenReturn(attempts); + + final ConnectionDataHistoryRequestBody requestBody = new ConnectionDataHistoryRequestBody() + .connectionId(connectionId) + .timezone(TIMEZONE_LOS_ANGELES); + final List actual = connectionsHandler.getConnectionDataHistory(requestBody); + + final List expected = generateEmptyConnectionDataHistoryReadList( + startTime.atZone(ZoneId.of(requestBody.getTimezone())).toLocalDate(), + endTime.atZone(ZoneId.of(requestBody.getTimezone())).toLocalDate(), + requestBody.getTimezone()); + expected.get(1).setBytes(Math.toIntExact(attempt1Bytes + attempt2Bytes)); + expected.get(2).setBytes(Math.toIntExact(attempt3Bytes)); + + assertEquals(actual, expected); + } + } } @@ -1813,7 +1942,7 @@ void testConnectionStatus() assertEquals(Enums.convertTo(JobStatus.FAILED, io.airbyte.api.model.generated.JobStatus.class), connectionStatus.getLastSyncJobStatus()); assertEquals(802L, connectionStatus.getLastSuccessfulSync()); assertEquals(true, connectionStatus.getIsRunning()); - assertEquals(null, connectionStatus.getNextSync()); + assertNull(connectionStatus.getNextSync()); } private AirbyteStreamAndConfiguration getStreamAndConfig(final String name, final AirbyteStreamConfiguration config) { @@ -1860,9 +1989,9 @@ class ApplySchemaChanges { CatalogHelpers.createAirbyteCatalog(SHOES, Field.of(SKU, JsonSchemaType.STRING)); private static final ConfiguredAirbyteCatalog configuredAirbyteCatalog = CatalogHelpers.createConfiguredAirbyteCatalog(SHOES, null, Field.of(SKU, JsonSchemaType.STRING)); - private static StandardSync standardSync; private static final String A_DIFFERENT_NAMESPACE = "a-different-namespace"; private static final String A_DIFFERENT_COLUMN = "a-different-column"; + private static StandardSync standardSync; @BeforeEach void setup() throws IOException, JsonValidationException, ConfigNotFoundException { @@ -1922,7 +2051,10 @@ void testAutoPropagateSchemaChange() throws IOException, ConfigNotFoundException expectedCatalog.getStreams() .add(new ConfiguredAirbyteStream().withStream(new io.airbyte.protocol.models.AirbyteStream().withName(A_DIFFERENT_STREAM) .withNamespace(A_DIFFERENT_NAMESPACE).withSupportedSyncModes(List.of(io.airbyte.protocol.models.SyncMode.FULL_REFRESH)) - .withDefaultCursorField(null)).withCursorField(null)); + .withDefaultCursorField(null)) + .withDestinationSyncMode(io.airbyte.protocol.models.DestinationSyncMode.OVERWRITE) + .withSyncMode(io.airbyte.protocol.models.SyncMode.FULL_REFRESH) + .withCursorField(null)); final ArgumentCaptor standardSyncArgumentCaptor = ArgumentCaptor.forClass(StandardSync.class); verify(configRepository).writeStandardSync(standardSyncArgumentCaptor.capture()); final StandardSync actualStandardSync = standardSyncArgumentCaptor.getValue(); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/OrganizationsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/OrganizationsHandlerTest.java index 08a0150daa9..6646f064302 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/OrganizationsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/OrganizationsHandlerTest.java @@ -83,17 +83,24 @@ void testGetOrganization() throws Exception { @Test void testUpdateOrganization() throws Exception { + // test updating org name and email final String newName = "new name"; + final String newEmail = "new email"; when(organizationPersistence.getOrganization(ORGANIZATION_ID_1)) .thenReturn(Optional.of(new Organization().withOrganizationId(ORGANIZATION_ID_1).withEmail(ORGANIZATION_EMAIL).withName(ORGANIZATION_NAME))); - when(organizationPersistence.updateOrganization(ORGANIZATION.withName(newName))) - .thenReturn(ORGANIZATION.withName(newName)); + when(organizationPersistence.updateOrganization(ORGANIZATION + .withName(newName) + .withEmail(newEmail))) + .thenReturn(ORGANIZATION.withName(newName).withEmail(newEmail)); final OrganizationRead result = - organizationsHandler.updateOrganization(new OrganizationUpdateRequestBody().organizationId(ORGANIZATION_ID_1).organizationName(newName)); + organizationsHandler.updateOrganization(new OrganizationUpdateRequestBody() + .organizationId(ORGANIZATION_ID_1) + .organizationName(newName) + .email(newEmail)); assertEquals(ORGANIZATION_ID_1, result.getOrganizationId()); assertEquals(newName, result.getOrganizationName()); - assertEquals(ORGANIZATION_EMAIL, result.getEmail()); + assertEquals(newEmail, result.getEmail()); } @Test diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java index 3585b1e4f91..499c6d6fe2a 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java @@ -61,6 +61,7 @@ import io.airbyte.api.model.generated.SourceUpdate; import io.airbyte.api.model.generated.StreamTransform; import io.airbyte.api.model.generated.StreamTransform.TransformTypeEnum; +import io.airbyte.api.model.generated.SyncMode; import io.airbyte.api.model.generated.SynchronousJobRead; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.features.EnvVariableFeatureFlags; @@ -1639,7 +1640,8 @@ void testAutoPropagateSchemaChangeAddStream() throws IOException, ConfigNotFound final io.airbyte.api.model.generated.AirbyteCatalog catalogWithDiff = CatalogConverter.toApi(Jsons.clone(airbyteCatalog), sourceVersion); - catalogWithDiff.addStreamsItem(new AirbyteStreamAndConfiguration().stream(new AirbyteStream().name(A_DIFFERENT_STREAM)) + catalogWithDiff.addStreamsItem(new AirbyteStreamAndConfiguration().stream(new AirbyteStream().name(A_DIFFERENT_STREAM) + .supportedSyncModes(List.of(SyncMode.FULL_REFRESH))) .config(new AirbyteStreamConfiguration().selected(true))); final UUID workspaceId = source.getWorkspaceId(); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java index 6fd46bdfc94..907875d31d6 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java @@ -7,7 +7,6 @@ import static io.airbyte.commons.server.handlers.helpers.AutoPropagateSchemaChangeHelper.extractStreamAndConfigPerStreamDescriptor; import static io.airbyte.commons.server.handlers.helpers.AutoPropagateSchemaChangeHelper.getUpdatedSchema; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -22,7 +21,6 @@ import io.airbyte.api.model.generated.StreamTransform; import io.airbyte.api.model.generated.SyncMode; import io.airbyte.commons.json.Jsons; -import io.airbyte.featureflag.AutoPropagateNewStreams; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.TestClient; import java.util.List; @@ -143,7 +141,6 @@ void applyAddNoFlag() { @Test void applyAdd() { - when(featureFlagClient.boolVariation(eq(AutoPropagateNewStreams.INSTANCE), any())).thenReturn(true); final JsonNode oldSchema = Jsons.deserialize(OLD_SCHEMA); final AirbyteCatalog oldAirbyteCatalog = createAirbyteCatalogWithSchema(NAME1, oldSchema); @@ -174,7 +171,6 @@ void applyAdd() { @Test void applyAddWithSourceDefinedCursor() { - when(featureFlagClient.boolVariation(eq(AutoPropagateNewStreams.INSTANCE), any())).thenReturn(true); final JsonNode oldSchema = Jsons.deserialize(OLD_SCHEMA); final AirbyteCatalog oldAirbyteCatalog = createAirbyteCatalogWithSchema(NAME1, oldSchema); @@ -206,7 +202,6 @@ void applyAddWithSourceDefinedCursor() { @Test void applyAddWithSourceDefinedCursorNoPrimaryKey() { - when(featureFlagClient.boolVariation(eq(AutoPropagateNewStreams.INSTANCE), any())).thenReturn(true); final JsonNode oldSchema = Jsons.deserialize(OLD_SCHEMA); final AirbyteCatalog oldAirbyteCatalog = createAirbyteCatalogWithSchema(NAME1, oldSchema); @@ -231,7 +226,6 @@ void applyAddWithSourceDefinedCursorNoPrimaryKey() { @Test void applyAddWithSourceDefinedCursorNoPrimaryKeyNoFullRefresh() { - when(featureFlagClient.boolVariation(eq(AutoPropagateNewStreams.INSTANCE), any())).thenReturn(true); final JsonNode oldSchema = Jsons.deserialize(OLD_SCHEMA); final AirbyteCatalog oldAirbyteCatalog = createAirbyteCatalogWithSchema(NAME1, oldSchema); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java index 09830e1457a..da1bf4aa08e 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java @@ -4,13 +4,13 @@ package io.airbyte.commons.server.support; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_IDS_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.DESTINATION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.JOB_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.OPERATION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.ORGANIZATION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.PERMISSION_ID_HEADER; -import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_DEFINITION_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_ID_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_IDS_HEADER; import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_ID_HEADER; @@ -65,6 +65,20 @@ void testResolvingFromConnectionId() throws JsonValidationException, ConfigNotFo assertEquals(List.of(workspaceId), result); } + @Test + void testResolvingFromConnectionIds() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID connectionId = UUID.randomUUID(); + final UUID connectionId2 = UUID.randomUUID(); + + final Map properties = Map.of(CONNECTION_IDS_HEADER, Jsons.serialize(List.of(connectionId.toString(), connectionId2.toString()))); + when(workspaceHelper.getWorkspaceForConnectionId(connectionId)).thenReturn(workspaceId); + when(workspaceHelper.getWorkspaceForConnectionId(connectionId2)).thenReturn(workspaceId); + + final List result = resolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId, workspaceId), result); + } + @Test void testResolvingFromSourceAndDestinationId() throws JsonValidationException, ConfigNotFoundException { final UUID workspaceId = UUID.randomUUID(); @@ -110,17 +124,6 @@ void testResolvingFromSourceId() throws JsonValidationException, ConfigNotFoundE assertEquals(List.of(workspaceId), result); } - @Test - void testResolvingFromSourceDefinitionId() throws JsonValidationException, ConfigNotFoundException { - final UUID workspaceId = UUID.randomUUID(); - final UUID sourceDefinitionId = UUID.randomUUID(); - final Map properties = Map.of(SOURCE_DEFINITION_ID_HEADER, sourceDefinitionId.toString()); - when(workspaceHelper.getWorkspaceForSourceId(sourceDefinitionId)).thenReturn(workspaceId); - - final List result = resolver.resolveWorkspace(properties); - assertEquals(List.of(workspaceId), result); - } - @Test void testResolvingFromOperationId() throws JsonValidationException, ConfigNotFoundException { final UUID workspaceId = UUID.randomUUID(); diff --git a/airbyte-commons-temporal-core/build.gradle b/airbyte-commons-temporal-core/build.gradle deleted file mode 100644 index 0b76d192bc1..00000000000 --- a/airbyte-commons-temporal-core/build.gradle +++ /dev/null @@ -1,21 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - id "org.jetbrains.kotlin.jvm" -} - -dependencies { - implementation libs.bundles.temporal - implementation libs.failsafe - - // We do not want dependency on databases from this library. - implementation project(':airbyte-commons') - implementation project(':airbyte-metrics:metrics-lib') - - testImplementation libs.assertj.core - testImplementation libs.bundles.junit - testImplementation libs.junit.pioneer - testImplementation libs.mockito.inline - testImplementation libs.temporal.testing - testRuntimeOnly libs.junit.jupiter.engine -} diff --git a/airbyte-commons-temporal-core/build.gradle.kts b/airbyte-commons-temporal-core/build.gradle.kts new file mode 100644 index 00000000000..1ba09a2244d --- /dev/null +++ b/airbyte-commons-temporal-core/build.gradle.kts @@ -0,0 +1,21 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + kotlin("jvm") +} + +dependencies { + implementation(libs.bundles.temporal) + implementation(libs.failsafe) + + // We do not want dependency on(databases from this library.) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-metrics:metrics-lib")) + + testImplementation(libs.assertj.core) + testImplementation(libs.bundles.junit) + testImplementation(libs.junit.pioneer) + testImplementation(libs.mockito.inline) + testImplementation(libs.temporal.testing) + testRuntimeOnly(libs.junit.jupiter.engine) +} diff --git a/airbyte-commons-temporal/build.gradle b/airbyte-commons-temporal/build.gradle deleted file mode 100644 index 03846c59c01..00000000000 --- a/airbyte-commons-temporal/build.gradle +++ /dev/null @@ -1,43 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.bundles.temporal - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-temporal-core') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-featureflag') - implementation project(':airbyte-metrics:metrics-lib') - implementation project(':airbyte-notification') - implementation project(':airbyte-persistence:job-persistence') - implementation libs.airbyte.protocol - implementation project(':airbyte-worker-models') - implementation project(':airbyte-api') - implementation project(':airbyte-json-validation') - - compileOnly libs.lombok - annotationProcessor libs.lombok - implementation libs.bundles.apache - implementation libs.failsafe - - testImplementation libs.temporal.testing - // Needed to be able to mock final class - testImplementation libs.mockito.inline - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer -} diff --git a/airbyte-commons-temporal/build.gradle.kts b/airbyte-commons-temporal/build.gradle.kts new file mode 100644 index 00000000000..e0b2bdf727a --- /dev/null +++ b/airbyte-commons-temporal/build.gradle.kts @@ -0,0 +1,43 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.bundles.temporal) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-temporal-core")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-featureflag")) + implementation(project(":airbyte-metrics:metrics-lib")) + implementation(project(":airbyte-notification")) + implementation(project(":airbyte-persistence:job-persistence")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-worker-models")) + implementation(project(":airbyte-api")) + implementation(project(":airbyte-json-validation")) + + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + implementation(libs.bundles.apache) + implementation(libs.failsafe) + + testImplementation(libs.temporal.testing) + // Needed to be able to mock final class) + testImplementation(libs.mockito.inline) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) +} diff --git a/airbyte-commons-with-dependencies/build.gradle b/airbyte-commons-with-dependencies/build.gradle deleted file mode 100644 index 210b0d5546a..00000000000 --- a/airbyte-commons-with-dependencies/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-temporal') - implementation project(':airbyte-config:config-models') - - implementation libs.guava - - compileOnly libs.lombok - annotationProcessor libs.lombok - - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - testImplementation libs.mockito.core - testImplementation libs.bundles.micronaut.test -} diff --git a/airbyte-commons-with-dependencies/build.gradle.kts b/airbyte-commons-with-dependencies/build.gradle.kts new file mode 100644 index 00000000000..ea9b05724a1 --- /dev/null +++ b/airbyte-commons-with-dependencies/build.gradle.kts @@ -0,0 +1,22 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-temporal")) + implementation(project(":airbyte-config:config-models")) + + implementation(libs.guava) + + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + + testImplementation(libs.mockito.core) + testImplementation(libs.bundles.micronaut.test) +} diff --git a/airbyte-commons-worker/build.gradle b/airbyte-commons-worker/build.gradle deleted file mode 100644 index 1d0517a36fd..00000000000 --- a/airbyte-commons-worker/build.gradle +++ /dev/null @@ -1,106 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - id "org.jetbrains.kotlin.jvm" - id "org.jetbrains.kotlin.kapt" -} - -configurations.all { - resolutionStrategy { - force libs.platform.testcontainers.postgresql - } -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - kapt libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.micronaut.http - implementation libs.kotlin.logging - implementation libs.micronaut.kotlin.extensions - - implementation libs.bundles.kubernetes.client - implementation 'com.auth0:java-jwt:3.19.2' - implementation libs.guava - implementation(libs.temporal.sdk) { - exclude module: 'guava' - } - implementation libs.apache.ant - implementation libs.apache.commons.text - implementation libs.bundles.datadog - implementation libs.commons.io - compileOnly libs.lombok - annotationProcessor libs.lombok - implementation libs.bundles.apache - implementation libs.bundles.log4j - implementation libs.failsafe.okhttp - implementation libs.google.cloud.storage - implementation libs.okhttp - implementation libs.aws.java.sdk.s3 - implementation libs.aws.java.sdk.sts - implementation libs.s3 - implementation libs.sts - - implementation project(':airbyte-api') - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-auth') - implementation project(':airbyte-commons-converters') - implementation project(':airbyte-commons-protocol') - implementation project(':airbyte-commons-temporal') - implementation project(':airbyte-commons-temporal-core') - implementation project(':airbyte-commons-with-dependencies') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-config:config-secrets') - implementation project(':airbyte-featureflag') - implementation project(':airbyte-json-validation') - implementation project(':airbyte-metrics:metrics-lib') - implementation project(':airbyte-persistence:job-persistence') - implementation libs.airbyte.protocol - implementation project(':airbyte-worker-models') - implementation libs.jakarta.validation.api - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - testAnnotationProcessor libs.jmh.annotations - - testImplementation libs.bundles.micronaut.test - testImplementation libs.json.path - testImplementation libs.mockito.inline - testImplementation(variantOf(libs.opentracing.util.test) { classifier('tests') }) - testImplementation libs.postgresql - testImplementation libs.platform.testcontainers.postgresql - testImplementation libs.jmh.core - testImplementation libs.jmh.annotations - testImplementation libs.docker.java - testImplementation libs.docker.java.transport.httpclient5 - testImplementation libs.reactor.test - - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - testImplementation libs.junit.pioneer -} - -test { - maxHeapSize = '4g' - - useJUnitPlatform { - excludeTags("cloud-storage") - } -} - -// The DuplicatesStrategy will be required while this module is mixture of kotlin and java _with_ lombok dependencies. -// Kapt, by default, runs all annotation processors and disables annotation processing by javac, however -// this default behavior breaks the lombok java annotation processor. To avoid lombok breaking, kapt has -// keepJavacAnnotationProcessors enabled, which causes duplicate META-INF files to be generated. -// Once lombok has been removed, this can also be removed. -tasks.withType(Jar).configureEach { - duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} diff --git a/airbyte-commons-worker/build.gradle.kts b/airbyte-commons-worker/build.gradle.kts new file mode 100644 index 00000000000..d9fae2ce492 --- /dev/null +++ b/airbyte-commons-worker/build.gradle.kts @@ -0,0 +1,107 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + kotlin("jvm") + kotlin("kapt") +} + +configurations.all { + resolutionStrategy { + force(libs.platform.testcontainers.postgresql) + } +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + kapt(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.micronaut.http) + implementation(libs.kotlin.logging) + implementation(libs.micronaut.kotlin.extensions) + + implementation(libs.bundles.kubernetes.client) + implementation("com.auth0:java-jwt:3.19.2") + implementation(libs.gson) + implementation(libs.guava) + implementation(libs.temporal.sdk) { + exclude(module = "guava") + } + implementation(libs.apache.ant) + implementation(libs.apache.commons.text) + implementation(libs.bundles.datadog) + implementation(libs.commons.io) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + implementation(libs.bundles.apache) + implementation(libs.bundles.log4j) + implementation(libs.failsafe.okhttp) + implementation(libs.google.cloud.storage) + implementation(libs.okhttp) + implementation(libs.aws.java.sdk.s3) + implementation(libs.aws.java.sdk.sts) + implementation(libs.s3) + implementation(libs.sts) + + implementation(project(":airbyte-api")) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-auth")) + implementation(project(":airbyte-commons-converters")) + implementation(project(":airbyte-commons-protocol")) + implementation(project(":airbyte-commons-temporal")) + implementation(project(":airbyte-commons-temporal-core")) + implementation(project(":airbyte-commons-with-dependencies")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-config:config-secrets")) + implementation(project(":airbyte-featureflag")) + implementation(project(":airbyte-json-validation")) + implementation(project(":airbyte-metrics:metrics-lib")) + implementation(project(":airbyte-persistence:job-persistence")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-worker-models")) + implementation(libs.jakarta.validation.api) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + testAnnotationProcessor(libs.jmh.annotations) + + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.json.path) + testImplementation(libs.mockito.inline) + testImplementation(variantOf(libs.opentracing.util.test) { classifier("tests") }) + testImplementation(libs.postgresql) + testImplementation(libs.platform.testcontainers.postgresql) + testImplementation(libs.jmh.core) + testImplementation(libs.jmh.annotations) + testImplementation(libs.docker.java) + testImplementation(libs.docker.java.transport.httpclient5) + testImplementation(libs.reactor.test) + + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + testImplementation(libs.junit.pioneer) +} + +tasks.named("test") { + maxHeapSize = "4g" + + useJUnitPlatform { + excludeTags("cloud-storage") + } +} + +// The DuplicatesStrategy will be required while this module is mixture of kotlin and java _with_ lombok dependencies.) +// Kapt, by default, runs all annotation(processors and disables annotation(processing by javac, however) +// this default behavior(breaks the lombok java annotation(processor. To avoid(lombok breaking, kapt(has) +// keepJavacAnnotationProcessors enabled, which causes duplicate META-INF files to be generated.) +// Once lombok has been removed, this can also be removed.) +tasks.withType().configureEach { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java index 2e0a66adec7..a8bc298bab3 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java @@ -17,6 +17,7 @@ import io.airbyte.commons.helper.DockerImageNameHelper; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.logging.LoggingHelper; import io.airbyte.commons.logging.LoggingHelper.Color; import io.airbyte.commons.logging.MdcScope; import io.airbyte.commons.logging.MdcScope.Builder; @@ -50,7 +51,7 @@ public class DbtTransformationRunner implements AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(DbtTransformationRunner.class); private static final String DBT_ENTRYPOINT_SH = "entrypoint.sh"; private static final MdcScope.Builder CONTAINER_LOG_MDC_BUILDER = new Builder() - .setLogPrefix("dbt") + .setLogPrefix(LoggingHelper.CUSTOM_TRANSFORMATION_LOGGER_PREFIX) .setPrefixColor(Color.PURPLE_BACKGROUND); private final ProcessFactory processFactory; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java index 4c5b1f54c5c..95bf70d8def 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorker.java @@ -28,6 +28,7 @@ import io.airbyte.protocol.models.AirbyteControlConnectorConfigMessage; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.internal.AirbyteStreamFactory; @@ -38,6 +39,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.StringJoiner; import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,6 +108,10 @@ public ConnectorJobOutput run(final StandardDiscoverCatalogInput discoverSchemaI } if (catalog.isPresent()) { + final String error = validateCatalog(catalog.get()); + if (!error.isEmpty()) { + WorkerUtils.throwWorkerException(error, process); + } final DiscoverCatalogResult result = AirbyteApiClient.retryWithJitter(() -> airbyteApiClient.getSourceApi() .writeDiscoverCatalogResult(buildSourceDiscoverSchemaWriteRequestBody(discoverSchemaInput, catalog.get())), @@ -124,6 +130,34 @@ public ConnectorJobOutput run(final StandardDiscoverCatalogInput discoverSchemaI } } + private String validateCatalog(final AirbyteCatalog persistenceCatalog) { + final StringJoiner streamsWithFaultySchema = new StringJoiner(","); + for (final AirbyteStream s : persistenceCatalog.getStreams()) { + if (s.getJsonSchema() != null && s.getJsonSchema().has("properties") && s.getJsonSchema().get("properties") != null) { + final JsonNode jsonSchema = s.getJsonSchema().get("properties"); + for (final String cursor : s.getDefaultCursorField()) { + if (!jsonSchema.has(cursor)) { + streamsWithFaultySchema.add( + String.format("Stream %s has declared cursor field %s but it's not part of the schema %s", s.getName(), cursor, + jsonSchema.toPrettyString())); + } + } + + for (final List pkey : s.getSourceDefinedPrimaryKey()) { + for (final String k : pkey) { + if (!jsonSchema.has(k)) { + streamsWithFaultySchema.add( + String.format("Stream %s has declared primary key field %s but it's not part of the schema %s", s.getName(), k, + jsonSchema.toPrettyString())); + } + } + } + } + } + + return streamsWithFaultySchema.toString(); + } + private SourceDiscoverSchemaWriteRequestBody buildSourceDiscoverSchemaWriteRequestBody(final StandardDiscoverCatalogInput discoverSchemaInput, final AirbyteCatalog catalog) { return new SourceDiscoverSchemaWriteRequestBody().catalog( diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java index e50f7d8159f..92aa4b28588 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java @@ -18,11 +18,13 @@ import io.airbyte.featureflag.Connection; import io.airbyte.featureflag.Context; import io.airbyte.featureflag.Destination; +import io.airbyte.featureflag.DestinationTimeoutSeconds; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.FieldSelectionEnabled; import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.RemoveValidationLimit; import io.airbyte.featureflag.ReplicationWorkerImpl; +import io.airbyte.featureflag.ShouldFailSyncOnDestinationTimeout; import io.airbyte.featureflag.Source; import io.airbyte.featureflag.SourceDefinition; import io.airbyte.featureflag.SourceType; @@ -238,11 +240,16 @@ private static HeartbeatTimeoutChaperone createHeartbeatTimeoutChaperone(final H private static DestinationTimeoutMonitor createDestinationTimeout(final FeatureFlagClient featureFlagClient, final ReplicationInput replicationInput, final MetricClient metricClient) { + Context context = new Multi(List.of(new Workspace(replicationInput.getWorkspaceId()), new Connection(replicationInput.getConnectionId()))); + boolean throwExceptionOnDestinationTimeout = featureFlagClient.boolVariation(ShouldFailSyncOnDestinationTimeout.INSTANCE, context); + int destinationTimeoutSeconds = featureFlagClient.intVariation(DestinationTimeoutSeconds.INSTANCE, context); + return new DestinationTimeoutMonitor( - featureFlagClient, replicationInput.getWorkspaceId(), replicationInput.getConnectionId(), - metricClient); + metricClient, + Duration.ofSeconds(destinationTimeoutSeconds), + throwExceptionOnDestinationTimeout); } /** diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/AirbyteMessageExtractor.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/AirbyteMessageExtractor.java new file mode 100644 index 00000000000..c8d47bab836 --- /dev/null +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/AirbyteMessageExtractor.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.helper; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.jooq.tools.StringUtils; + +public class AirbyteMessageExtractor { + + public static Optional getCatalogStreamFromMessage(final ConfiguredAirbyteCatalog catalog, + final String name, + final String namespace) { + return catalog.getStreams() + .stream() + .filter(configuredStream -> StringUtils.equals(configuredStream.getStream().getNamespace(), namespace) + && StringUtils.equals(configuredStream.getStream().getName(), name)) + .findFirst(); + } + + public static Optional getCatalogStreamFromMessage(final ConfiguredAirbyteCatalog catalog, + final AirbyteRecordMessage message) { + return getCatalogStreamFromMessage(catalog, message.getStream(), message.getNamespace()); + } + + public static List> getPks(final Optional catalogStream) { + return catalogStream + .map(stream -> stream.getPrimaryKey()) + .orElse(new ArrayList<>()); + } + + public static boolean containsNonNullPK(final List pks, final JsonNode data) { + return Jsons.navigateTo(data, pks) != null; + } + +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/GsonPksExtractor.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/GsonPksExtractor.java new file mode 100644 index 00000000000..f0acb2d9659 --- /dev/null +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/GsonPksExtractor.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.helper; + +import com.google.common.base.Joiner; +import com.google.gson.Gson; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import jakarta.inject.Singleton; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * When the line are too big for jackson, we are failine the sync with a too big line error. This + * error is not helping our users to identify which line is causing an issue. In order to be able to + * identify which line is faulty, we need to use another library to parse the line and extract the + * Pks from it. This class is doing that using GSON. + */ +@Singleton +public class GsonPksExtractor { + + private final Gson gson = new Gson(); + + public String extractPks(final ConfiguredAirbyteCatalog catalog, + final String line) { + final Map jsonLine = (Map) gson.fromJson(line, Map.class).get("record"); + + final String name = (String) jsonLine.get("stream"); + final String namespace = (String) jsonLine.get("namespace"); + + final Optional catalogStream = AirbyteMessageExtractor.getCatalogStreamFromMessage(catalog, + name, + namespace); + final List> pks = AirbyteMessageExtractor.getPks(catalogStream); + + final Map pkValues = new HashMap<>(); + + pks.forEach(pk -> { + final String value = navigateTo(jsonLine.get("data"), pk); + pkValues.put(String.join(".", pk), value == null ? "[MISSING]" : value); + }); + + return mapToString(pkValues); + } + + private String mapToString(final Map map) { + return Joiner.on(",").withKeyValueSeparator("=").join(map); + } + + private String navigateTo(final Object json, final List keys) { + Object jsonInternal = json; + for (final String key : keys) { + if (jsonInternal == null) { + return null; + } + jsonInternal = ((Map) jsonInternal).get(key); + } + return jsonInternal == null ? null : jsonInternal.toString(); + } + +} diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/BasicAirbyteMessageValidator.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/BasicAirbyteMessageValidator.java index 82a04cd04ab..40c08e9d536 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/BasicAirbyteMessageValidator.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/BasicAirbyteMessageValidator.java @@ -4,7 +4,15 @@ package io.airbyte.workers.internal; +import com.google.common.collect.Iterables; import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; +import io.airbyte.workers.helper.AirbyteMessageExtractor; +import io.airbyte.workers.internal.exception.SourceException; +import java.util.List; import java.util.Optional; /** @@ -26,7 +34,9 @@ */ public class BasicAirbyteMessageValidator { - static Optional validate(AirbyteMessage message) { + static Optional validate(final AirbyteMessage message, + final Optional catalog, + final boolean failMissingPks) { if (message.getType() == null) { return Optional.empty(); } @@ -47,6 +57,30 @@ final var record = message.getRecord(); if (record.getStream() == null || record.getData() == null) { return Optional.empty(); } + if (failMissingPks && catalog.isPresent()) { + final Optional catalogStream = AirbyteMessageExtractor.getCatalogStreamFromMessage(catalog.get(), record); + + if (catalogStream.isEmpty()) { + throw new SourceException(String.format("Missing catalog stream for the stream (namespace: %s, name: %s", + record.getStream(), record.getNamespace())); + } else if (catalogStream.get().getSyncMode().equals(SyncMode.INCREMENTAL) + && catalogStream.get().getDestinationSyncMode().equals(DestinationSyncMode.APPEND_DEDUP)) { + // required PKs + final List> pksList = AirbyteMessageExtractor.getPks(catalogStream); + if (pksList.isEmpty()) { + throw new SourceException(String.format("Primary keys not found in catalog for the stream (namespace: %s, name: %s", + record.getStream(), record.getNamespace())); + } + + final boolean containsAtLeastOneNonNullPk = Iterables.tryFind(pksList, + pks -> AirbyteMessageExtractor.containsNonNullPK(pks, record.getData())).isPresent(); + + if (!containsAtLeastOneNonNullPk) { + throw new SourceException(String.format("All the defined primary keys are null, the primary keys are: %s", + String.join(", ", pksList.stream().map(pks -> String.join(".", pks)).toList()))); + } + } + } } case LOG -> { if (message.getLog() == null) { diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java index 92cd29334e6..418061637b1 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java @@ -11,6 +11,7 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.logging.LoggingHelper; import io.airbyte.commons.logging.LoggingHelper.Color; import io.airbyte.commons.logging.MdcScope; import io.airbyte.commons.logging.MdcScope.Builder; @@ -21,6 +22,7 @@ import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.workers.WorkerUtils; import io.airbyte.workers.exception.WorkerException; +import io.airbyte.workers.helper.GsonPksExtractor; import io.airbyte.workers.process.IntegrationLauncher; import java.io.BufferedWriter; import java.io.IOException; @@ -42,7 +44,7 @@ public class DefaultAirbyteDestination implements AirbyteDestination { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultAirbyteDestination.class); public static final MdcScope.Builder CONTAINER_LOG_MDC_BUILDER = new Builder() - .setLogPrefix("destination") + .setLogPrefix(LoggingHelper.DESTINATION_LOGGER_PREFIX) .setPrefixColor(Color.YELLOW_BACKGROUND); static final Set IGNORED_EXIT_CODES = Set.of( 0, // Normal exit @@ -65,8 +67,13 @@ public class DefaultAirbyteDestination implements AirbyteDestination { @VisibleForTesting public DefaultAirbyteDestination(final IntegrationLauncher integrationLauncher, final DestinationTimeoutMonitor destinationTimeoutMonitor) { this(integrationLauncher, - VersionedAirbyteStreamFactory.noMigrationVersionedAirbyteStreamFactory(LOGGER, CONTAINER_LOG_MDC_BUILDER, Optional.empty(), - Runtime.getRuntime().maxMemory(), false), + VersionedAirbyteStreamFactory.noMigrationVersionedAirbyteStreamFactory( + LOGGER, + CONTAINER_LOG_MDC_BUILDER, + Optional.empty(), + Runtime.getRuntime().maxMemory(), + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration(false, false, false), + new GsonPksExtractor()), new DefaultAirbyteMessageBufferedWriterFactory(), new DefaultProtocolSerializer(), destinationTimeoutMonitor); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java index 556015dab1d..323e0fc43ae 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java @@ -16,6 +16,7 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.logging.LoggingHelper; import io.airbyte.commons.logging.LoggingHelper.Color; import io.airbyte.commons.logging.MdcScope; import io.airbyte.commons.logging.MdcScope.Builder; @@ -51,7 +52,7 @@ public class DefaultAirbyteSource implements AirbyteSource { ); public static final MdcScope.Builder CONTAINER_LOG_MDC_BUILDER = new Builder() - .setLogPrefix("source") + .setLogPrefix(LoggingHelper.SOURCE_LOGGER_PREFIX) .setPrefixColor(Color.BLUE_BACKGROUND); private final IntegrationLauncher integrationLauncher; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DestinationTimeoutMonitor.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DestinationTimeoutMonitor.java index 042cf94e122..1496bb9fb94 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DestinationTimeoutMonitor.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DestinationTimeoutMonitor.java @@ -7,17 +7,12 @@ import static java.lang.Thread.sleep; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.featureflag.Connection; -import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.ShouldFailSyncOnDestinationTimeout; -import io.airbyte.featureflag.Workspace; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import java.time.Duration; -import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -48,38 +43,38 @@ public class DestinationTimeoutMonitor implements AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(DestinationTimeoutMonitor.class); private static final Duration POLL_INTERVAL = Duration.ofMinutes(1); - private static final Duration TIMEOUT = Duration.ofHours(2); private final AtomicReference currentAcceptCallStartTime = new AtomicReference<>(null); private final AtomicReference currentNotifyEndOfInputCallStartTime = new AtomicReference<>(null); - private final FeatureFlagClient featureFlagClient; private final UUID workspaceId; private ExecutorService lazyExecutorService; private final UUID connectionId; private final MetricClient metricClient; private final Duration pollInterval; private final Duration timeout; + private final boolean throwExceptionOnTimeout; @VisibleForTesting - public DestinationTimeoutMonitor(final FeatureFlagClient featureFlagClient, - final UUID workspaceId, + public DestinationTimeoutMonitor(final UUID workspaceId, final UUID connectionId, final MetricClient metricClient, - final Duration pollInterval, - final Duration timeout) { - this.featureFlagClient = featureFlagClient; + final Duration timeout, + final boolean throwExceptionOnTimeout, + final Duration pollInterval) { this.workspaceId = workspaceId; this.connectionId = connectionId; this.metricClient = metricClient; - this.pollInterval = pollInterval; this.timeout = timeout; + this.throwExceptionOnTimeout = throwExceptionOnTimeout; + this.pollInterval = pollInterval; } - public DestinationTimeoutMonitor(final FeatureFlagClient featureFlagClient, - final UUID workspaceId, + public DestinationTimeoutMonitor(final UUID workspaceId, final UUID connectionId, - final MetricClient metricClient) { - this(featureFlagClient, workspaceId, connectionId, metricClient, POLL_INTERVAL, TIMEOUT); + final MetricClient metricClient, + final Duration timeout, + final boolean throwExceptionOnTimeout) { + this(workspaceId, connectionId, metricClient, timeout, throwExceptionOnTimeout, POLL_INTERVAL); } /** @@ -190,8 +185,7 @@ public void resetNotifyEndOfInputTimer() { } private void onTimeout(final CompletableFuture runnableFuture) { - if (featureFlagClient.boolVariation(ShouldFailSyncOnDestinationTimeout.INSTANCE, - new Multi(List.of(new Workspace(workspaceId), new Connection(connectionId))))) { + if (throwExceptionOnTimeout) { runnableFuture.cancel(true); throw new TimeoutException("Destination has timed out"); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java index 022f2ea3c1d..e681807f6a2 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java @@ -26,6 +26,7 @@ import io.airbyte.protocol.models.AirbyteLogMessage; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.workers.helper.GsonPksExtractor; import java.io.BufferedReader; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -55,6 +56,8 @@ @SuppressWarnings("PMD.MoreThanOneLogger") public class VersionedAirbyteStreamFactory implements AirbyteStreamFactory { + public record InvalidLineFailureConfiguration(boolean failTooLongRecords, boolean failMissingPks, boolean printLongRecordPks) {} + private static final Logger LOGGER = LoggerFactory.getLogger(VersionedAirbyteStreamFactory.class); private static final double MAX_SIZE_RATIO = 0.8; private static final long DEFAULT_MEMORY_LIMIT = Runtime.getRuntime().maxMemory(); @@ -67,7 +70,7 @@ public class VersionedAirbyteStreamFactory implements AirbyteStreamFactory { // Given that BufferedReader::reset fails if we try to reset if we go past its buffer size, this // buffer has to be big enough to contain our longest spec and whatever messages get emitted before // the SPEC. - private static final int BUFFER_READ_AHEAD_LIMIT = 32000; + private static final int BUFFER_READ_AHEAD_LIMIT = 2 * 1024 * 1024; // 2 megabytes private static final int MESSAGES_LOOK_AHEAD_FOR_DETECTION = 10; private static final String TYPE_FIELD_NAME = "type"; private static final int MAXIMUM_CHARACTERS_ALLOWED = 5_000_000; @@ -88,7 +91,8 @@ public class VersionedAirbyteStreamFactory implements AirbyteStreamFactory { private boolean shouldDetectVersion = false; - private final boolean failTooLongRecords; + private final InvalidLineFailureConfiguration invalidLineFailureConfiguration; + private final GsonPksExtractor gsonPksExtractor; /** * In some cases, we know the stream will never emit messages that need to be migrated. This is @@ -99,7 +103,7 @@ public class VersionedAirbyteStreamFactory implements AirbyteStreamFactory { @VisibleForTesting public static VersionedAirbyteStreamFactory noMigrationVersionedAirbyteStreamFactory(final boolean failTooLongRecords) { return noMigrationVersionedAirbyteStreamFactory(LOGGER, MdcScope.DEFAULT_BUILDER, Optional.empty(), Runtime.getRuntime().maxMemory(), - failTooLongRecords); + new InvalidLineFailureConfiguration(failTooLongRecords, false, false), new GsonPksExtractor()); } /** @@ -112,7 +116,8 @@ public static VersionedAirbyteStreamFactory noMigrationVersionedAirbyteStreamFac final MdcScope.Builder mdcBuilder, final Optional> clazz, final long maxMemory, - final boolean failTooLongRecords) { + final InvalidLineFailureConfiguration conf, + final GsonPksExtractor gsonPksExtractor) { final AirbyteMessageSerDeProvider provider = new AirbyteMessageSerDeProvider( List.of(new AirbyteMessageV0Deserializer(), new AirbyteMessageV1Deserializer()), List.of(new AirbyteMessageV0Serializer(), new AirbyteMessageV1Serializer())); @@ -126,7 +131,7 @@ public static VersionedAirbyteStreamFactory noMigrationVersionedAirbyteStreamFac new AirbyteProtocolVersionedMigratorFactory(airbyteMessageMigrator, configuredAirbyteCatalogMigrator); return new VersionedAirbyteStreamFactory<>(provider, fac, AirbyteProtocolVersion.DEFAULT_AIRBYTE_PROTOCOL_VERSION, Optional.empty(), logger, - mdcBuilder, clazz, maxMemory, failTooLongRecords); + mdcBuilder, clazz, maxMemory, conf, gsonPksExtractor); } public VersionedAirbyteStreamFactory(final AirbyteMessageSerDeProvider serDeProvider, @@ -135,9 +140,10 @@ public VersionedAirbyteStreamFactory(final AirbyteMessageSerDeProvider serDeProv final Optional configuredAirbyteCatalog, final MdcScope.Builder containerLogMdcBuilder, final Optional> exceptionClass, - final boolean failTooLongRecords) { + final InvalidLineFailureConfiguration invalidLineFailureConfiguration, + final GsonPksExtractor gsonPksExtractor) { this(serDeProvider, migratorFactory, protocolVersion, configuredAirbyteCatalog, LOGGER, containerLogMdcBuilder, exceptionClass, - Runtime.getRuntime().maxMemory(), failTooLongRecords); + Runtime.getRuntime().maxMemory(), invalidLineFailureConfiguration, gsonPksExtractor); } public VersionedAirbyteStreamFactory(final AirbyteMessageSerDeProvider serDeProvider, @@ -145,9 +151,10 @@ public VersionedAirbyteStreamFactory(final AirbyteMessageSerDeProvider serDeProv final Version protocolVersion, final Optional configuredAirbyteCatalog, final Optional> exceptionClass, - final boolean failTooLongRecords) { + final InvalidLineFailureConfiguration invalidLineFailureConfiguration, + final GsonPksExtractor gsonPksExtractor) { this(serDeProvider, migratorFactory, protocolVersion, configuredAirbyteCatalog, DEFAULT_LOGGER, DEFAULT_MDC_SCOPE, exceptionClass, - DEFAULT_MEMORY_LIMIT, failTooLongRecords); + DEFAULT_MEMORY_LIMIT, invalidLineFailureConfiguration, gsonPksExtractor); } public VersionedAirbyteStreamFactory(final AirbyteMessageSerDeProvider serDeProvider, @@ -158,19 +165,21 @@ public VersionedAirbyteStreamFactory(final AirbyteMessageSerDeProvider serDeProv final MdcScope.Builder containerLogMdcBuilder, final Optional> exceptionClass, final long maxMemory, - final boolean failTooLongRecords) { + final InvalidLineFailureConfiguration invalidLineFailureConfiguration, + final GsonPksExtractor gsonPksExtractor) { // TODO AirbyteProtocolPredicate needs to be updated to be protocol version aware this.logger = logger; this.containerLogMdcBuilder = containerLogMdcBuilder; this.exceptionClass = exceptionClass; this.maxMemory = maxMemory; + this.gsonPksExtractor = gsonPksExtractor; Preconditions.checkNotNull(protocolVersion); this.serDeProvider = serDeProvider; this.migratorFactory = migratorFactory; this.configuredAirbyteCatalog = configuredAirbyteCatalog; this.initializeForProtocolVersion(protocolVersion); - this.failTooLongRecords = failTooLongRecords; + this.invalidLineFailureConfiguration = invalidLineFailureConfiguration; } /** @@ -346,7 +355,7 @@ protected Stream toAirbyteMessage(final String line) { Optional m = deserializer.deserializeExact(line); if (m.isPresent()) { - m = BasicAirbyteMessageValidator.validate(m.get()); + m = BasicAirbyteMessageValidator.validate(m.get(), configuredAirbyteCatalog, invalidLineFailureConfiguration.failMissingPks); if (m.isEmpty()) { logger.error("Validation failed: {}", Jsons.serialize(line)); @@ -374,13 +383,18 @@ protected Stream toAirbyteMessage(final String line) { *

* */ - private void handleCannotDeserialize(String line) { - try (final var mdcScope = containerLogMdcBuilder.build()) { + private void handleCannotDeserialize(final String line) { + try (final MdcScope ignored = containerLogMdcBuilder.build()) { if (line.length() >= MAXIMUM_CHARACTERS_ALLOWED) { MetricClientFactory.getMetricClient().count(OssMetricsRegistry.LINE_SKIPPED_TOO_LONG, 1); MetricClientFactory.getMetricClient().distribution(OssMetricsRegistry.TOO_LONG_LINES_DISTRIBUTION, line.length()); - LOGGER.error("[LINE TOO BIG] line is too big with size: " + line.length()); - if (failTooLongRecords) { + if (invalidLineFailureConfiguration.printLongRecordPks) { + LOGGER.error("[LARGE RECORD] A record is too long with size: " + line.length()); + configuredAirbyteCatalog.ifPresent( + airbyteCatalog -> LOGGER + .error("[LARGE RECORD] The primary keys of the long record are: " + gsonPksExtractor.extractPks(airbyteCatalog, line))); + } + if (invalidLineFailureConfiguration.failTooLongRecords) { if (exceptionClass.isPresent()) { throwExceptionClass("One record is too big and can't be processed, the sync will be failed"); } else { diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index f2645dace1b..981cb84c97d 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -15,6 +15,7 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.logging.LoggingHelper; import io.airbyte.commons.logging.LoggingHelper.Color; import io.airbyte.commons.logging.MdcScope; import io.airbyte.commons.logging.MdcScope.Builder; @@ -51,7 +52,7 @@ public class DefaultNormalizationRunner implements NormalizationRunner { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultNormalizationRunner.class); private static final MdcScope.Builder CONTAINER_LOG_MDC_BUILDER = new Builder() - .setLogPrefix("normalization") + .setLogPrefix(LoggingHelper.NORMALIZATION_LOGGER_PREFIX) .setPrefixColor(Color.GREEN_BACKGROUND); private final String normalizationIntegrationType; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncherFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncherFactory.java index d416b16ed28..15b86066ec0 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncherFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncherFactory.java @@ -11,12 +11,15 @@ import io.airbyte.commons.protocol.VersionedProtocolSerializer; import io.airbyte.config.SyncResourceRequirements; import io.airbyte.featureflag.Connection; +import io.airbyte.featureflag.FailMissingPks; import io.airbyte.featureflag.FailSyncIfTooBig; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.PrintLongRecordPks; import io.airbyte.featureflag.Workspace; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.workers.helper.GsonPksExtractor; import io.airbyte.workers.internal.AirbyteDestination; import io.airbyte.workers.internal.AirbyteSource; import io.airbyte.workers.internal.AirbyteStreamFactory; @@ -44,17 +47,20 @@ public class AirbyteIntegrationLauncherFactory { private final AirbyteProtocolVersionedMigratorFactory migratorFactory; private final FeatureFlags featureFlags; private final FeatureFlagClient featureFlagClient; + private final GsonPksExtractor gsonPksExtractor; public AirbyteIntegrationLauncherFactory(final ProcessFactory processFactory, final AirbyteMessageSerDeProvider serDeProvider, final AirbyteProtocolVersionedMigratorFactory migratorFactory, final FeatureFlags featureFlags, - final FeatureFlagClient featureFlagClient) { + final FeatureFlagClient featureFlagClient, + final GsonPksExtractor gsonPksExtractor) { this.processFactory = processFactory; this.serDeProvider = serDeProvider; this.migratorFactory = migratorFactory; this.featureFlags = featureFlags; this.featureFlagClient = featureFlagClient; + this.gsonPksExtractor = gsonPksExtractor; } /** @@ -103,9 +109,22 @@ public AirbyteSource createAirbyteSource(final IntegrationLauncherConfig sourceL new Connection(sourceLauncherConfig.getConnectionId()), new Workspace(sourceLauncherConfig.getWorkspaceId())))); + final boolean failMissingPks = featureFlagClient.boolVariation(FailMissingPks.INSTANCE, + new Multi(List.of( + new Connection(sourceLauncherConfig.getConnectionId()), + new Workspace(sourceLauncherConfig.getWorkspaceId())))); + + final boolean printLongRecordPks = featureFlagClient.boolVariation(PrintLongRecordPks.INSTANCE, + new Multi(List.of( + new Connection(sourceLauncherConfig.getConnectionId()), + new Workspace(sourceLauncherConfig.getWorkspaceId())))); + return new DefaultAirbyteSource(sourceLauncher, getStreamFactory(sourceLauncherConfig, configuredAirbyteCatalog, SourceException.class, DefaultAirbyteSource.CONTAINER_LOG_MDC_BUILDER, - failTooLongRecords), + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration( + failTooLongRecords, + failMissingPks, + printLongRecordPks)), heartbeatMonitor, getProtocolSerializer(sourceLauncherConfig), featureFlags); @@ -125,8 +144,11 @@ public AirbyteDestination createAirbyteDestination(final IntegrationLauncherConf final IntegrationLauncher destinationLauncher = createIntegrationLauncher(destinationLauncherConfig, syncResourceRequirements); return new DefaultAirbyteDestination(destinationLauncher, - getStreamFactory(destinationLauncherConfig, configuredAirbyteCatalog, DestinationException.class, - DefaultAirbyteDestination.CONTAINER_LOG_MDC_BUILDER, false), + getStreamFactory(destinationLauncherConfig, + configuredAirbyteCatalog, + DestinationException.class, + DefaultAirbyteDestination.CONTAINER_LOG_MDC_BUILDER, + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration(false, false, false)), new VersionedAirbyteMessageBufferedWriterFactory(serDeProvider, migratorFactory, destinationLauncherConfig.getProtocolVersion(), Optional.of(configuredAirbyteCatalog)), getProtocolSerializer(destinationLauncherConfig), destinationTimeoutMonitor); @@ -140,9 +162,9 @@ private AirbyteStreamFactory getStreamFactory(final IntegrationLauncherConfig la final ConfiguredAirbyteCatalog configuredAirbyteCatalog, final Class exceptionClass, final MdcScope.Builder mdcScopeBuilder, - final boolean failTooLongRecords) { + final VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration invalidLineFailureConfiguration) { return new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, launcherConfig.getProtocolVersion(), - Optional.of(configuredAirbyteCatalog), mdcScopeBuilder, Optional.of(exceptionClass), failTooLongRecords); + Optional.of(configuredAirbyteCatalog), mdcScopeBuilder, Optional.of(exceptionClass), invalidLineFailureConfiguration, gsonPksExtractor); } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/WorkloadApiWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/WorkloadApiWorker.java index 98cba829d12..ffe7dec7416 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/WorkloadApiWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/WorkloadApiWorker.java @@ -4,6 +4,12 @@ package io.airbyte.workers.sync; +import static io.airbyte.config.helpers.LogClientSingleton.fullLogPath; + +import io.airbyte.api.client.AirbyteApiClient; +import io.airbyte.api.client.invoker.generated.ApiException; +import io.airbyte.api.client.model.generated.ConnectionIdRequestBody; +import io.airbyte.api.client.model.generated.Geography; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ReplicationOutput; import io.airbyte.featureflag.Connection; @@ -37,6 +43,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import org.openapitools.client.infrastructure.ClientException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +57,7 @@ public class WorkloadApiWorker implements Worker TERMINAL_STATUSES = Set.of(WorkloadStatus.CANCELLED, WorkloadStatus.FAILURE, WorkloadStatus.SUCCESS); private final DocumentStoreClient documentStoreClient; private final OrchestratorNameGenerator orchestratorNameGenerator; + private final AirbyteApiClient apiClient; private final WorkloadApi workloadApi; private final WorkloadIdGenerator workloadIdGenerator; private final ReplicationActivityInput input; @@ -58,12 +66,14 @@ public class WorkloadApiWorker implements Worker { try { invocation.callRealMethod(); @@ -1033,11 +1026,11 @@ void testDestinationNotifyEndOfInputTimeout() throws Exception { .thenReturn(new ReplicationFeatureFlags(true, 0)); destinationTimeoutMonitor = spy(new DestinationTimeoutMonitor( - featureFlagClient, UUID.randomUUID(), UUID.randomUUID(), metricClient, Duration.ofSeconds(1), + true, Duration.ofSeconds(1))); destination = new SimpleTimeoutMonitoredDestination(destinationTimeoutMonitor); @@ -1051,8 +1044,6 @@ void testDestinationNotifyEndOfInputTimeout() throws Exception { return null; }).when(destinationTimeoutMonitor).resetNotifyEndOfInputTimer(); - when(featureFlagClient.boolVariation(eq(ShouldFailSyncOnDestinationTimeout.INSTANCE), any(Context.class))).thenReturn(true); - doAnswer(invocation -> { try { invocation.callRealMethod(); @@ -1081,11 +1072,11 @@ void testDestinationTimeoutWithCloseFailure() throws Exception { .thenReturn(new ReplicationFeatureFlags(true, 0)); destinationTimeoutMonitor = spy(new DestinationTimeoutMonitor( - featureFlagClient, UUID.randomUUID(), UUID.randomUUID(), metricClient, Duration.ofSeconds(1), + true, Duration.ofSeconds(1))); destination = spy(new SimpleTimeoutMonitoredDestination(destinationTimeoutMonitor)); @@ -1104,8 +1095,6 @@ void testDestinationTimeoutWithCloseFailure() throws Exception { return null; }).when(destinationTimeoutMonitor).resetNotifyEndOfInputTimer(); - when(featureFlagClient.boolVariation(eq(ShouldFailSyncOnDestinationTimeout.INSTANCE), any(Context.class))).thenReturn(true); - doAnswer(invocation -> { try { invocation.callRealMethod(); diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java index 2202948cf95..c81acb52b63 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java @@ -158,10 +158,11 @@ public void executeOneSync() throws InterruptedException { new AirbyteControlMessageEventListener(connectorConfigUpdater), new AirbyteStreamStatusMessageEventListener(streamStatusTracker)); final DestinationTimeoutMonitor destinationTimeoutMonitor = new DestinationTimeoutMonitor( - featureFlagClient, workspaceID, UUID.randomUUID(), - new NotImplementedMetricClient()); + new NotImplementedMetricClient(), + Duration.ofMinutes(120), + false); final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper = mock(ReplicationAirbyteMessageEventPublishingHelper.class); doAnswer((e) -> { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/GsonPksExtractorTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/GsonPksExtractorTest.java new file mode 100644 index 00000000000..b2529481fcc --- /dev/null +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/GsonPksExtractorTest.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.helper; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; +import java.util.List; +import org.junit.jupiter.api.Test; + +class GsonPksExtractorTest { + + private final GsonPksExtractor gsonPksExtractor = new GsonPksExtractor(); + + private static final String STREAM_NAME = "test"; + private static final String FIELD_1_NAME = "field1"; + + @Test + void testNonExistingPk() { + final var record = """ + { + "record": { + "stream": "test", + "data": { + "field": "value" + } + } + } + """; + + final var configureAirbyteCatalog = getCatalogWithPk(STREAM_NAME, List.of(List.of(FIELD_1_NAME))); + + assertEquals("field1=[MISSING]", gsonPksExtractor.extractPks(configureAirbyteCatalog, record)); + } + + @Test + void testExistingPkLevel1() { + final var record = """ + { + "record": { + "stream": "test", + "data": { + "field1": "value" + } + } + } + """; + + final var configureAirbyteCatalog = getCatalogWithPk(STREAM_NAME, List.of(List.of(FIELD_1_NAME))); + + assertEquals("field1=value", gsonPksExtractor.extractPks(configureAirbyteCatalog, record)); + } + + @Test + void testExistingPkLevel2() { + final var record = """ + { + "record": { + "stream": "test", + "data": { + "level1": { + "field1": "value" + } + } + } + } + """; + + final var configureAirbyteCatalog = getCatalogWithPk(STREAM_NAME, List.of(List.of("level1", FIELD_1_NAME))); + + assertEquals("level1.field1=value", gsonPksExtractor.extractPks(configureAirbyteCatalog, record)); + } + + @Test + void testExistingPkMultipleKey() { + final var record = """ + { + "record": { + "stream": "test", + "data": { + "field1": "value", + "field2": "value2" + } + } + } + """; + + final var configureAirbyteCatalog = getCatalogWithPk(STREAM_NAME, List.of(List.of(FIELD_1_NAME), List.of("field2"))); + + assertEquals("field1=value,field2=value2", gsonPksExtractor.extractPks(configureAirbyteCatalog, record)); + } + + @Test + void testExistingVeryLongRecord() { + final StringBuilder longStringBuilder = new StringBuilder(5_000_000); + for (int i = 0; i < 50_000_000; i++) { + longStringBuilder.append("a"); + } + final var record = String.format(""" + { + "record": { + "stream": "test", + "data": { + "field1": "value", + "veryLongField": "%s" + } + } + } + """, longStringBuilder); + + final var configureAirbyteCatalog = getCatalogWithPk(STREAM_NAME, List.of(List.of(FIELD_1_NAME))); + + assertEquals("field1=value", gsonPksExtractor.extractPks(configureAirbyteCatalog, record)); + } + + private ConfiguredAirbyteCatalog getCatalogWithPk(final String streamName, + final List> pksList) { + return new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream() + .withName(streamName)) + .withPrimaryKey(pksList) + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP))); + } + +} diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/BasicAirbyteMessageValidatorTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/BasicAirbyteMessageValidatorTest.java index b3876631448..298a522b395 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/BasicAirbyteMessageValidatorTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/BasicAirbyteMessageValidatorTest.java @@ -5,30 +5,42 @@ package io.airbyte.workers.internal; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.Config; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.SyncMode; +import io.airbyte.workers.internal.exception.SourceException; import io.airbyte.workers.test_utils.AirbyteMessageUtils; +import java.util.List; import java.util.Optional; -import org.junit.Test; +import org.junit.jupiter.api.Test; -public class BasicAirbyteMessageValidatorTest { +class BasicAirbyteMessageValidatorTest { + + private static final String DATA_KEY_1 = "field_1"; + private static final String STREAM_1 = "stream_1"; + private static final String DATA_VALUE = "green"; @Test void testObviousInvalid() { final Optional bad = Jsons.tryDeserializeExact("{}", AirbyteMessage.class); - final var m = BasicAirbyteMessageValidator.validate(bad.get()); + final var m = BasicAirbyteMessageValidator.validate(bad.get(), Optional.empty(), false); assertTrue(m.isEmpty()); } @Test void testValidRecord() { - final AirbyteMessage rec = AirbyteMessageUtils.createRecordMessage("stream_1", "field_1", "green"); + final AirbyteMessage rec = AirbyteMessageUtils.createRecordMessage(STREAM_1, DATA_KEY_1, DATA_VALUE); - final var m = BasicAirbyteMessageValidator.validate(rec); + final var m = BasicAirbyteMessageValidator.validate(rec, Optional.empty(), false); assertTrue(m.isPresent()); assertEquals(rec, m.get()); } @@ -37,7 +49,7 @@ void testValidRecord() { void testSubtleInvalidRecord() { final Optional bad = Jsons.tryDeserializeExact("{\"type\": \"RECORD\", \"record\": {}}", AirbyteMessage.class); - final var m = BasicAirbyteMessageValidator.validate(bad.get()); + final var m = BasicAirbyteMessageValidator.validate(bad.get(), Optional.empty(), false); assertTrue(m.isEmpty()); } @@ -45,7 +57,7 @@ void testSubtleInvalidRecord() { void testValidState() { final AirbyteMessage rec = AirbyteMessageUtils.createStateMessage(1); - final var m = BasicAirbyteMessageValidator.validate(rec); + final var m = BasicAirbyteMessageValidator.validate(rec, Optional.empty(), false); assertTrue(m.isPresent()); assertEquals(rec, m.get()); } @@ -54,7 +66,7 @@ void testValidState() { void testSubtleInvalidState() { final Optional bad = Jsons.tryDeserializeExact("{\"type\": \"STATE\", \"control\": {}}", AirbyteMessage.class); - final var m = BasicAirbyteMessageValidator.validate(bad.get()); + final var m = BasicAirbyteMessageValidator.validate(bad.get(), Optional.empty(), false); assertTrue(m.isEmpty()); } @@ -62,7 +74,7 @@ void testSubtleInvalidState() { void testValidControl() { final AirbyteMessage rec = AirbyteMessageUtils.createConfigControlMessage(new Config(), 1000.0); - final var m = BasicAirbyteMessageValidator.validate(rec); + final var m = BasicAirbyteMessageValidator.validate(rec, Optional.empty(), false); assertTrue(m.isPresent()); assertEquals(rec, m.get()); } @@ -71,8 +83,98 @@ void testValidControl() { void testSubtleInvalidControl() { final Optional bad = Jsons.tryDeserializeExact("{\"type\": \"CONTROL\", \"state\": {}}", AirbyteMessage.class); - final var m = BasicAirbyteMessageValidator.validate(bad.get()); + final var m = BasicAirbyteMessageValidator.validate(bad.get(), Optional.empty(), false); assertTrue(m.isEmpty()); } + @Test + void testValidPk() { + final AirbyteMessage bad = AirbyteMessageUtils.createRecordMessage(STREAM_1, DATA_KEY_1, DATA_VALUE); + + final var m = BasicAirbyteMessageValidator.validate(bad, Optional.of( + getCatalogWithPk(STREAM_1, List.of(List.of(DATA_KEY_1)))), true); + assertTrue(m.isPresent()); + } + + @Test + void testValidPkWithOneMissingPk() { + final AirbyteMessage bad = AirbyteMessageUtils.createRecordMessage(STREAM_1, DATA_KEY_1, DATA_VALUE); + + final var m = BasicAirbyteMessageValidator.validate(bad, Optional.of( + getCatalogWithPk(STREAM_1, List.of(List.of(DATA_KEY_1), List.of("not_field_1")))), true); + assertTrue(m.isPresent()); + } + + @Test + void testNotIncrementalDedup() { + final AirbyteMessage bad = AirbyteMessageUtils.createRecordMessage(STREAM_1, DATA_KEY_1, DATA_VALUE); + + var m = BasicAirbyteMessageValidator.validate(bad, Optional.of( + getCatalogNonIncremental(STREAM_1)), + true); + assertTrue(m.isPresent()); + + m = BasicAirbyteMessageValidator.validate(bad, Optional.of( + getCatalogNonIncrementalDedup(STREAM_1)), + true); + assertTrue(m.isPresent()); + } + + @Test + void testInvalidPk() { + final AirbyteMessage bad = AirbyteMessageUtils.createRecordMessage(STREAM_1, DATA_KEY_1, DATA_VALUE); + + assertThrows(SourceException.class, () -> BasicAirbyteMessageValidator.validate(bad, Optional.of( + getCatalogWithPk(STREAM_1, List.of(List.of("not_field_1")))), true)); + } + + @Test + void testValidPkInAnotherStream() { + final AirbyteMessage bad = AirbyteMessageUtils.createRecordMessage(STREAM_1, DATA_KEY_1, DATA_VALUE); + + assertThrows(SourceException.class, () -> BasicAirbyteMessageValidator.validate(bad, Optional.of( + getCatalogWithPk("stream_2", List.of(List.of(DATA_KEY_1)))), true)); + } + + @Test + void testInvalidPkWithoutFlag() { + final AirbyteMessage bad = AirbyteMessageUtils.createRecordMessage(STREAM_1, DATA_KEY_1, DATA_VALUE); + + final var m = BasicAirbyteMessageValidator.validate(bad, Optional.of( + getCatalogWithPk(STREAM_1, List.of(List.of("not_field_1")))), false); + assertTrue(m.isPresent()); + } + + private ConfiguredAirbyteCatalog getCatalogWithPk(final String streamName, + final List> pksList) { + return new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream() + .withName(streamName)) + .withPrimaryKey(pksList) + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP))); + } + + private ConfiguredAirbyteCatalog getCatalogNonIncrementalDedup(final String streamName) { + return new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream() + .withName(streamName)) + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND))); + } + + private ConfiguredAirbyteCatalog getCatalogNonIncremental(final String streamName) { + return new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream() + .withName(streamName)) + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.APPEND))); + } + } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/DestinationTimeoutMonitorTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/DestinationTimeoutMonitorTest.java index db72277bfbf..68c8bcc739d 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/DestinationTimeoutMonitorTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/DestinationTimeoutMonitorTest.java @@ -14,12 +14,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import io.airbyte.featureflag.Context; -import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.ShouldFailSyncOnDestinationTimeout; -import io.airbyte.featureflag.TestClient; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; import java.time.Duration; @@ -29,18 +24,17 @@ class DestinationTimeoutMonitorTest { - private final FeatureFlagClient featureFlagClient = mock(TestClient.class); private final MetricClient metricClient = mock(MetricClient.class); @Test void testNoTimeout() { DestinationTimeoutMonitor destinationTimeoutMonitor = new DestinationTimeoutMonitor( - featureFlagClient, UUID.randomUUID(), UUID.randomUUID(), metricClient, - Duration.ofMillis(500), - Duration.ofMinutes(5)); + Duration.ofMinutes(5), + true, + Duration.ofMillis(500)); destinationTimeoutMonitor.startAcceptTimer(); destinationTimeoutMonitor.startNotifyEndOfInputTimer(); @@ -60,15 +54,13 @@ void testNoTimeout() { @Test void testAcceptTimeout() { DestinationTimeoutMonitor destinationTimeoutMonitor = new DestinationTimeoutMonitor( - featureFlagClient, UUID.randomUUID(), UUID.randomUUID(), metricClient, Duration.ofSeconds(1), + true, Duration.ofSeconds(1)); - when(featureFlagClient.boolVariation(eq(ShouldFailSyncOnDestinationTimeout.INSTANCE), any(Context.class))).thenReturn(true); - destinationTimeoutMonitor.startAcceptTimer(); assertThrows( @@ -88,15 +80,13 @@ void testAcceptTimeout() { @Test void testNotifyEndOfInputTimeout() { DestinationTimeoutMonitor destinationTimeoutMonitor = new DestinationTimeoutMonitor( - featureFlagClient, UUID.randomUUID(), UUID.randomUUID(), metricClient, Duration.ofSeconds(1), + true, Duration.ofSeconds(1)); - when(featureFlagClient.boolVariation(eq(ShouldFailSyncOnDestinationTimeout.INSTANCE), any(Context.class))).thenReturn(true); - destinationTimeoutMonitor.startNotifyEndOfInputTimer(); assertThrows( @@ -114,13 +104,13 @@ void testNotifyEndOfInputTimeout() { } @Test - void testTimeoutNoExceptionWhenFeatureFlagDisabled() { + void testTimeoutNoException() { DestinationTimeoutMonitor destinationTimeoutMonitor = new DestinationTimeoutMonitor( - featureFlagClient, UUID.randomUUID(), UUID.randomUUID(), metricClient, Duration.ofSeconds(1), + false, Duration.ofSeconds(1)); destinationTimeoutMonitor.startAcceptTimer(); diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactoryTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactoryTest.java index 3cdd7d47389..d67f9efa70f 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactoryTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactoryTest.java @@ -25,6 +25,7 @@ import io.airbyte.commons.version.Version; import io.airbyte.protocol.models.AirbyteLogMessage; import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.workers.helper.GsonPksExtractor; import io.airbyte.workers.test_utils.AirbyteMessageUtils; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -55,6 +56,8 @@ class VersionedAirbyteStreamFactoryTest { AirbyteMessageSerDeProvider serDeProvider; AirbyteProtocolVersionedMigratorFactory migratorFactory; + private final GsonPksExtractor gsonPksExtractor = mock(GsonPksExtractor.class); + @Nested @DisplayName("Test Correct AirbyteMessage Parsing Behavior") class ParseMessages { @@ -162,7 +165,8 @@ void testFailsSize() { final Stream messageStream = VersionedAirbyteStreamFactory - .noMigrationVersionedAirbyteStreamFactory(logger, new Builder(), Optional.of(RuntimeException.class), 1L, false) + .noMigrationVersionedAirbyteStreamFactory(logger, new Builder(), Optional.of(RuntimeException.class), 1L, + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration(false, false, false), gsonPksExtractor) .create(bufferedReader); assertThrows(RuntimeException.class, () -> messageStream.toList()); @@ -183,10 +187,11 @@ void testMalformedRecordShouldOnlyDebugLog(final String invalidRecord) { verify(logger).debug(invalidRecord); } - private VersionedAirbyteStreamFactory getFactory(final boolean failTooLongMessage) { + private VersionedAirbyteStreamFactory getFactory(final boolean failTooLongMessage, final boolean failMissingPks) { return VersionedAirbyteStreamFactory .noMigrationVersionedAirbyteStreamFactory(logger, new Builder(), Optional.of(RuntimeException.class), 100000L, - failTooLongMessage); + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration(failTooLongMessage, failMissingPks, false), + gsonPksExtractor); } private static final String VALID_MESSAGE_TEMPLATE = @@ -197,26 +202,26 @@ private VersionedAirbyteStreamFactory getFactory(final boolean failTooLongMessag @Test void testToAirbyteMessageValid() { final String messageLine = String.format(VALID_MESSAGE_TEMPLATE, "hello"); - Assertions.assertThat(getFactory(false).toAirbyteMessage(messageLine)).hasSize(1); + Assertions.assertThat(getFactory(false, false).toAirbyteMessage(messageLine)).hasSize(1); } @Test void testToAirbyteMessageRandomLog() { - Assertions.assertThat(getFactory(false).toAirbyteMessage("I should not be send on the same channel than the airbyte messages")) + Assertions.assertThat(getFactory(false, false).toAirbyteMessage("I should not be send on the same channel than the airbyte messages")) .isEmpty(); } @Test void testToAirbyteMessageMixedUpRecordShouldOnlyDebugLog() { final String messageLine = "It shouldn't be here" + String.format(VALID_MESSAGE_TEMPLATE, "hello"); - getFactory(false).toAirbyteMessage(messageLine); + getFactory(false, false).toAirbyteMessage(messageLine); verify(logger).debug(messageLine); } @Test void testToAirbyteMessageMixedUpRecordFailureDisable() { final String messageLine = "It shouldn't be here" + String.format(VALID_MESSAGE_TEMPLATE, "hello"); - Assertions.assertThat(getFactory(false).toAirbyteMessage(messageLine)).isEmpty(); + Assertions.assertThat(getFactory(false, false).toAirbyteMessage(messageLine)).isEmpty(); } @Test @@ -226,7 +231,7 @@ void testToAirbyteMessageVeryLongMessageFail() { longStringBuilder.append("a"); } final String messageLine = String.format(VALID_MESSAGE_TEMPLATE, longStringBuilder); - assertThrows(RuntimeException.class, () -> getFactory(true).toAirbyteMessage(messageLine)); + assertThrows(RuntimeException.class, () -> getFactory(true, false).toAirbyteMessage(messageLine)); } @Test @@ -236,14 +241,16 @@ void testToAirbyteMessageVeryLongMessageDontFail() { longStringBuilder.append("a"); } final String messageLine = String.format(VALID_MESSAGE_TEMPLATE, longStringBuilder); - Assertions.assertThat(getFactory(false).toAirbyteMessage(messageLine)).isEmpty(); + Assertions.assertThat(getFactory(false, false).toAirbyteMessage(messageLine)).isEmpty(); } private Stream stringToMessageStream(final String inputString) { final InputStream inputStream = new ByteArrayInputStream(inputString.getBytes(StandardCharsets.UTF_8)); final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); return VersionedAirbyteStreamFactory - .noMigrationVersionedAirbyteStreamFactory(logger, new Builder(), Optional.of(RuntimeException.class), 100000L, false) + .noMigrationVersionedAirbyteStreamFactory(logger, new Builder(), Optional.of(RuntimeException.class), 100000L, + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration(false, false, false), + gsonPksExtractor) .create(bufferedReader); } @@ -275,7 +282,9 @@ void beforeEach() { void testCreate() { final Version initialVersion = new Version("0.1.2"); final VersionedAirbyteStreamFactory streamFactory = - new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion, Optional.empty(), Optional.empty(), false); + new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion, Optional.empty(), Optional.empty(), + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration(false, false, false), + gsonPksExtractor); final BufferedReader bufferedReader = new BufferedReader(new StringReader("")); streamFactory.create(bufferedReader); @@ -287,8 +296,10 @@ void testCreate() { void testCreateWithVersionDetection() { final Version initialVersion = new Version("0.0.0"); final VersionedAirbyteStreamFactory streamFactory = - new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion, Optional.empty(), Optional.empty(), false) - .withDetectVersion(true); + new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion, Optional.empty(), Optional.empty(), + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration(false, false, false), + gsonPksExtractor) + .withDetectVersion(true); final BufferedReader bufferedReader = getBuffereredReader("version-detection/logs-with-version.jsonl"); @@ -302,8 +313,10 @@ void testCreateWithVersionDetection() { void testCreateWithVersionDetectionFallback() { final Version initialVersion = new Version("0.0.6"); final VersionedAirbyteStreamFactory streamFactory = - new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion, Optional.empty(), Optional.empty(), false) - .withDetectVersion(true); + new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion, Optional.empty(), Optional.empty(), + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration(false, false, false), + gsonPksExtractor) + .withDetectVersion(true); final BufferedReader bufferedReader = getBuffereredReader("version-detection/logs-without-version.jsonl"); @@ -317,8 +330,10 @@ void testCreateWithVersionDetectionFallback() { void testCreateWithVersionDetectionWithoutSpecMessage() { final Version initialVersion = new Version("0.0.1"); final VersionedAirbyteStreamFactory streamFactory = - new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion, Optional.empty(), Optional.empty(), false) - .withDetectVersion(true); + new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion, Optional.empty(), Optional.empty(), + new VersionedAirbyteStreamFactory.InvalidLineFailureConfiguration(false, false, false), + gsonPksExtractor) + .withDetectVersion(true); final BufferedReader bufferedReader = getBuffereredReader("version-detection/logs-without-spec-message.jsonl"); diff --git a/airbyte-commons/build.gradle b/airbyte-commons/build.gradle deleted file mode 100644 index ccd42f04933..00000000000 --- a/airbyte-commons/build.gradle +++ /dev/null @@ -1,45 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - id "de.undercouch.download" version "5.4.0" -} - -dependencies { - implementation libs.bundles.jackson - implementation libs.guava - implementation libs.bundles.slf4j - implementation libs.commons.io - implementation libs.bundles.apache - implementation libs.google.cloud.storage - implementation libs.bundles.log4j - implementation libs.airbyte.protocol - - compileOnly libs.lombok - annotationProcessor libs.lombok - - - // this dependency is an exception to the above rule because it is only used INTERNALLY to the commons library. - implementation 'com.jayway.jsonpath:json-path:2.7.0' - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer -} - -airbyte { - spotless { - excludes = ["src/main/resources/seed/specs_secrets_mask.yaml"] - } -} - -def downloadSpecSecretMask = tasks.register("downloadSpecSecretMask", Download) { - src 'https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml' - dest new File(projectDir, 'src/main/resources/seed/specs_secrets_mask.yaml') - overwrite true -} - -tasks.named("processResources") { - dependsOn(downloadSpecSecretMask) -} diff --git a/airbyte-commons/build.gradle.kts b/airbyte-commons/build.gradle.kts new file mode 100644 index 00000000000..4d5808c7839 --- /dev/null +++ b/airbyte-commons/build.gradle.kts @@ -0,0 +1,46 @@ +import de.undercouch.gradle.tasks.download.Download + +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + id("de.undercouch.download") version "5.4.0" +} + +dependencies { + implementation(libs.bundles.jackson) + implementation(libs.guava) + implementation(libs.bundles.slf4j) + implementation(libs.commons.io) + implementation(libs.bundles.apache) + implementation(libs.google.cloud.storage) + implementation(libs.bundles.log4j) + implementation(libs.airbyte.protocol) + + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + // this dependency is an exception to the above rule because it is only used INTERNALLY to the commons library. + implementation("com.jayway.jsonpath:json-path:2.7.0") + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) +} + +airbyte { + spotless { + excludes = listOf("src/main/resources/seed/specs_secrets_mask.yaml") + } +} + +val downloadSpecSecretMask = tasks.register("downloadSpecSecretMask") { + src("https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml") + dest(File(projectDir, "src/main/resources/seed/specs_secrets_mask.yaml")) + overwrite(true) +} + +tasks.named("processResources") { + dependsOn(downloadSpecSecretMask) +} diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java index c14b55a3b6b..6375f782025 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java @@ -11,23 +11,21 @@ */ public class LoggingHelper { + public static final String CUSTOM_TRANSFORMATION_LOGGER_PREFIX = "dbt"; + public static final String DESTINATION_LOGGER_PREFIX = "destination"; + public static final String NORMALIZATION_LOGGER_PREFIX = "normalization"; + public static final String SOURCE_LOGGER_PREFIX = "source"; + public static final String PLATFORM_LOGGER_PREFIX = "platform"; + /** * Color of log line. */ public enum Color { - BLACK("\u001b[30m"), - RED("\u001b[31m"), - GREEN("\u001b[32m"), - YELLOW("\u001b[33m"), - BLUE("\u001b[34m"), - MAGENTA("\u001b[35m"), - CYAN("\u001b[36m"), - WHITE("\u001b[37m"), BLUE_BACKGROUND("\u001b[44m"), // source YELLOW_BACKGROUND("\u001b[43m"), // destination GREEN_BACKGROUND("\u001b[42m"), // normalization - CYAN_BACKGROUND("\u001b[46m"), // container runner + CYAN_BACKGROUND("\u001b[46m"), // platform applications PURPLE_BACKGROUND("\u001b[45m"); // dbt private final String ansi; @@ -51,4 +49,8 @@ public static String applyColor(final Color color, final String msg) { return color.getCode() + msg + RESET; } + public static String platformLogSource() { + return applyColor(Color.CYAN_BACKGROUND, PLATFORM_LOGGER_PREFIX); + } + } diff --git a/airbyte-config/config-models/build.gradle b/airbyte-config/config-models/build.gradle deleted file mode 100644 index a39fca86624..00000000000 --- a/airbyte-config/config-models/build.gradle +++ /dev/null @@ -1,85 +0,0 @@ -import org.jsonschema2pojo.SourceType - -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - id "com.github.eirnym.js2p" - id "org.jetbrains.kotlin.jvm" -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) - implementation libs.bundles.jackson - implementation libs.spotbugs.annotations - implementation libs.guava - compileOnly libs.lombok - annotationProcessor libs.lombok - implementation libs.google.cloud.storage - implementation libs.aws.java.sdk.s3 - implementation libs.aws.java.sdk.sts - implementation libs.s3 - implementation libs.sts - implementation libs.bundles.apache - - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation project(':airbyte-json-validation') - implementation libs.airbyte.protocol - implementation libs.commons.io - implementation project(':airbyte-commons') - - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - testImplementation libs.junit.pioneer -} - -jsonSchema2Pojo { - sourceType = SourceType.YAMLSCHEMA - source = files("${sourceSets.main.output.resourcesDir}/types") - targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') - - targetPackage = 'io.airbyte.config' - useLongIntegers = true - - removeOldOutput = true - - generateBuilders = true - includeConstructors = false - includeSetters = true - serializable = true -} - -test { - useJUnitPlatform { - excludeTags 'log4j2-config', 'logger-client' - } -} - -tasks.named("compileKotlin") { - dependsOn tasks.named("generateJsonSchema2Pojo") -} - -tasks.register("log4j2IntegrationTest", Test) { - useJUnitPlatform { - includeTags 'log4j2-config' - } - testLogging { - events "passed", "skipped", "failed" - } -} - -tasks.register("logClientsIntegrationTest", Test) { - useJUnitPlatform { - includeTags 'logger-client' - } - testLogging { - events "passed", "skipped", "failed" - } -} diff --git a/airbyte-config/config-models/build.gradle.kts b/airbyte-config/config-models/build.gradle.kts new file mode 100644 index 00000000000..e6c0a0d75ff --- /dev/null +++ b/airbyte-config/config-models/build.gradle.kts @@ -0,0 +1,86 @@ +import org.gradle.api.tasks.testing.logging.TestLogEvent +import org.jsonschema2pojo.SourceType + +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + id("com.github.eirnym.js2p") + kotlin("jvm") +} + +dependencies { + annotationProcessor(libs.bundles.micronaut.annotation.processor) + api(libs.bundles.micronaut.annotation) + + implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) + implementation(libs.bundles.jackson) + implementation(libs.spotbugs.annotations) + implementation(libs.guava) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + implementation(libs.google.cloud.storage) + implementation(libs.aws.java.sdk.s3) + implementation(libs.aws.java.sdk.sts) + implementation(libs.s3) + implementation(libs.sts) + implementation(libs.bundles.apache) + + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + implementation(project(":airbyte-json-validation")) + implementation(libs.airbyte.protocol) + implementation(libs.commons.io) + implementation(project(":airbyte-commons")) + + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + testImplementation(libs.junit.pioneer) +} + +jsonSchema2Pojo { + setSourceType(SourceType.YAMLSCHEMA.name) + setSource(files("${sourceSets["main"].output.resourcesDir}/types")) + targetDirectory = file("$buildDir/generated/src/gen/java/") + + targetPackage = "io.airbyte.config" + useLongIntegers = true + + removeOldOutput = true + + generateBuilders = true + includeConstructors = false + includeSetters = true + serializable = true +} + +tasks.named("test") { + useJUnitPlatform { + excludeTags("log4j2-config", "logger-client") + } +} + +tasks.named("compileKotlin") { + dependsOn(tasks.named("generateJsonSchema2Pojo")) +} + +tasks.register("log4j2IntegrationTest") { + useJUnitPlatform { + includeTags("log4j2-config") + } + testLogging { + events = setOf(TestLogEvent.PASSED, TestLogEvent.SKIPPED, TestLogEvent.FAILED) + } +} + +tasks.register("logClientsIntegrationTest") { + useJUnitPlatform { + includeTags("logger-client") + } + testLogging { + events = setOf(TestLogEvent.PASSED, TestLogEvent.SKIPPED, TestLogEvent.FAILED) + } +} diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java index 0bc9e03694a..a1c93a6f5fe 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java @@ -189,7 +189,7 @@ public void setJobMdc(final WorkerEnvironment workerEnvironment, final LogConfig if (shouldUseLocalLogs(workerEnvironment)) { LOGGER.debug("Setting docker job mdc"); if (path != null) { - final String resolvedPath = path.resolve(LogClientSingleton.LOG_FILENAME).toString(); + final String resolvedPath = fullLogPath(path); MDC.put(LogClientSingleton.JOB_LOG_PATH_MDC_KEY, resolvedPath); } else { MDC.remove(LogClientSingleton.JOB_LOG_PATH_MDC_KEY); @@ -198,7 +198,7 @@ public void setJobMdc(final WorkerEnvironment workerEnvironment, final LogConfig LOGGER.debug("Setting kube job mdc"); createCloudClientIfNull(logConfigs); if (path != null) { - MDC.put(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + MDC.put(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY, fullLogPath(path)); } else { MDC.remove(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY); } @@ -223,9 +223,13 @@ public void setWorkspaceMdc(final WorkerEnvironment workerEnvironment, final Log } } + public static String fullLogPath(final Path rootPath) { + return rootPath.resolve(LogClientSingleton.LOG_FILENAME).toString(); + } + // This method should cease to exist here and become a property on the enum instead // TODO handle this as part of refactor https://github.com/airbytehq/airbyte/issues/7545 - private static boolean shouldUseLocalLogs(final WorkerEnvironment workerEnvironment) { + public static boolean shouldUseLocalLogs(final WorkerEnvironment workerEnvironment) { return workerEnvironment.equals(WorkerEnvironment.DOCKER); } diff --git a/airbyte-config/config-models/src/main/resources/types/ApiKey.yaml b/airbyte-config/config-models/src/main/resources/types/ApiKey.yaml new file mode 100644 index 00000000000..57a0d8af43d --- /dev/null +++ b/airbyte-config/config-models/src/main/resources/types/ApiKey.yaml @@ -0,0 +1,23 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/ApiKey.yaml +title: Api Key +description: An API key for a user or non-user entity i.e. an organization +type: object +required: + - name +additionalProperties: false +properties: + id: + type: string + description: An ID that uniquely identifies the API key in the downstream service. Is used for deletion. + name: + description: Caption name for the Api Key + type: string + apiKey: + description: The API key, only returned on creation + type: string + createdOn: + type: string + description: A date string in ISO 8601 format (e.g. 2021-01-01T00:00:00Z) that the key was created. + format: string diff --git a/airbyte-config/config-persistence/build.gradle b/airbyte-config/config-persistence/build.gradle deleted file mode 100644 index 0ee07e31974..00000000000 --- a/airbyte-config/config-persistence/build.gradle +++ /dev/null @@ -1,57 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - id "java-test-fixtures" -} - -configurations.all { - exclude group: 'io.micronaut.flyway' -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-protocol') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:specs') - implementation project(':airbyte-data') - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-db:jooq') - implementation project(':airbyte-featureflag') - implementation project(':airbyte-json-validation') - implementation libs.airbyte.protocol - implementation project(':airbyte-metrics:metrics-lib') - implementation libs.bundles.apache - implementation libs.google.cloud.storage - implementation libs.commons.io - - testImplementation 'org.hamcrest:hamcrest-all:1.3' - testImplementation libs.platform.testcontainers.postgresql - testImplementation libs.flyway.core - testImplementation libs.mockito.inline - testImplementation project(':airbyte-test-utils') - - integrationTestImplementation project(':airbyte-config:config-persistence') - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - testImplementation libs.junit.pioneer - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - - testFixturesApi libs.jackson.databind - testFixturesApi libs.guava - testFixturesApi project(':airbyte-json-validation') - testFixturesApi project(':airbyte-commons') - testFixturesApi project(':airbyte-config:config-models') - testFixturesApi project(':airbyte-config:config-secrets') - testFixturesApi libs.airbyte.protocol - testFixturesApi libs.lombok - testFixturesAnnotationProcessor libs.lombok -} diff --git a/airbyte-config/config-persistence/build.gradle.kts b/airbyte-config/config-persistence/build.gradle.kts new file mode 100644 index 00000000000..26aa4221b19 --- /dev/null +++ b/airbyte-config/config-persistence/build.gradle.kts @@ -0,0 +1,57 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + `java-test-fixtures` +} + +configurations.all { + exclude(group = "io.micronaut.flyway") +} + +dependencies { + annotationProcessor(libs.bundles.micronaut.annotation.processor) + api(libs.bundles.micronaut.annotation) + + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-protocol")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:specs")) + implementation(project(":airbyte-data")) + implementation(project(":airbyte-db:db-lib")) + implementation(project(":airbyte-db:jooq")) + implementation(project(":airbyte-featureflag")) + implementation(project(":airbyte-json-validation")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-metrics:metrics-lib")) + implementation(libs.bundles.apache) + implementation(libs.google.cloud.storage) + implementation(libs.commons.io) + + testImplementation("org.hamcrest:hamcrest-all:1.3") + testImplementation(libs.platform.testcontainers.postgresql) + testImplementation(libs.flyway.core) + testImplementation(libs.mockito.inline) + testImplementation(project(":airbyte-test-utils")) + + integrationTestImplementation(project(":airbyte-config:config-persistence")) + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + testImplementation(libs.junit.pioneer) + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + + testFixturesApi(libs.jackson.databind) + testFixturesApi(libs.guava) + testFixturesApi(project(":airbyte-json-validation")) + testFixturesApi(project(":airbyte-commons")) + testFixturesApi(project(":airbyte-config:config-models")) + testFixturesApi(project(":airbyte-config:config-secrets")) + testFixturesApi(libs.airbyte.protocol) + testFixturesApi(libs.lombok) + testFixturesAnnotationProcessor(libs.lombok) +} diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index b06c69d88c4..d9e171d7ee1 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -2093,27 +2093,6 @@ public Optional getActorDefinitionVersion(final UUID act return actorDefinitionService.getActorDefinitionVersion(actorDefinitionId, dockerImageTag); } - // /** - // * Get the actor definition version associated with an actor definition and a docker image tag. - // * - // * @param actorDefinitionId - actor definition id - // * @param dockerImageTag - docker image tag - // * @param ctx database context - // * @return actor definition version if there is an entry in the DB already for this version, - // * otherwise an empty optional - // * @throws IOException - you never know when you io - // */ - // public Optional getActorDefinitionVersion(final UUID actorDefinitionId, - // final String dockerImageTag, final DSLContext ctx) { - // return ctx.selectFrom(Tables.ACTOR_DEFINITION_VERSION) - // .where(Tables.ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID.eq(actorDefinitionId) - // .and(Tables.ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG.eq(dockerImageTag))) - // .fetch() - // .stream() - // .findFirst() - // .map(DbConverter::buildActorDefinitionVersion); - // } - /** * Get an actor definition version by ID. * diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java index e4a77f41b4f..49ed41e5f17 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java @@ -254,30 +254,28 @@ private List listInstanceAdminPermissions(final DSLContext ctx) return records.stream().map(record -> buildUserPermissionFromRecord(record)).collect(Collectors.toList()); } - private UserPermission getUserInstanceAdminPermission(final DSLContext ctx, final UUID userId) { - var record = ctx.select(USER.ID, USER.NAME, USER.EMAIL, USER.DEFAULT_WORKSPACE_ID, PERMISSION.ID, PERMISSION.PERMISSION_TYPE) + public Boolean isUserInstanceAdmin(final UUID userId) throws IOException { + return this.database.query(ctx -> isUserInstanceAdmin(ctx, userId)); + } + + private Boolean isUserInstanceAdmin(final DSLContext ctx, final UUID userId) { + return ctx.fetchExists(select() .from(PERMISSION) - .join(USER) - .on(PERMISSION.USER_ID.eq(USER.ID)) .where(PERMISSION.PERMISSION_TYPE.eq(io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.instance_admin)) - .and(PERMISSION.USER_ID.eq(userId)) - .fetchOne(); - if (record == null) { - return null; - } - return buildUserPermissionFromRecord(record); + .and(PERMISSION.USER_ID.eq(userId))); } - /** - * Check and get instance_admin permission for a user. - * - * @param userId user id - * @return UserPermission User details with instance_admin permission, null if user does not have - * instance_admin role. - * @throws IOException if there is an issue while interacting with the db. - */ - public UserPermission getUserInstanceAdminPermission(final UUID userId) throws IOException { - return this.database.query(ctx -> getUserInstanceAdminPermission(ctx, userId)); + public Boolean isAuthUserInstanceAdmin(final String authUserId) throws IOException { + return this.database.query(ctx -> isAuthUserInstanceAdmin(ctx, authUserId)); + } + + private Boolean isAuthUserInstanceAdmin(final DSLContext ctx, final String authUserId) { + return ctx.fetchExists(select() + .from(PERMISSION) + .join(USER) + .on(PERMISSION.USER_ID.eq(USER.ID)) + .where(PERMISSION.PERMISSION_TYPE.eq(io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.instance_admin)) + .and(USER.AUTH_USER_ID.eq(authUserId))); } public PermissionType findPermissionTypeForUserAndWorkspace(final UUID workspaceId, final String authUserId) diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java index 6dbd6cc6eb6..ddd594dcec6 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java @@ -200,6 +200,32 @@ void listPermissionsForOrganizationTest() throws Exception { } } + @Test + void isUserInstanceAdmin() throws IOException { + final User user1 = MockData.users().get(0); + Assertions.assertEquals(user1.getUserId(), MockData.permission1.getUserId()); + Assertions.assertEquals(MockData.permission1.getPermissionType(), PermissionType.INSTANCE_ADMIN); + Assertions.assertTrue(permissionPersistence.isUserInstanceAdmin(user1.getUserId())); + + final User user2 = MockData.users().get(1); + Assertions.assertEquals(user2.getUserId(), MockData.permission2.getUserId()); + Assertions.assertNotEquals(MockData.permission2.getPermissionType(), PermissionType.INSTANCE_ADMIN); + Assertions.assertFalse(permissionPersistence.isUserInstanceAdmin(user2.getUserId())); + } + + @Test + void isAuthUserInstanceAdmin() throws IOException { + final User user1 = MockData.users().get(0); + Assertions.assertEquals(user1.getUserId(), MockData.permission1.getUserId()); + Assertions.assertEquals(MockData.permission1.getPermissionType(), PermissionType.INSTANCE_ADMIN); + Assertions.assertTrue(permissionPersistence.isAuthUserInstanceAdmin(user1.getAuthUserId())); + + final User user2 = MockData.users().get(1); + Assertions.assertEquals(user2.getUserId(), MockData.permission2.getUserId()); + Assertions.assertNotEquals(MockData.permission2.getPermissionType(), PermissionType.INSTANCE_ADMIN); + Assertions.assertFalse(permissionPersistence.isAuthUserInstanceAdmin(user2.getAuthUserId())); + } + @Nested class WritePermission { diff --git a/airbyte-config/config-secrets/build.gradle.kts b/airbyte-config/config-secrets/build.gradle.kts index 94ac8b4f012..15fb7abee95 100644 --- a/airbyte-config/config-secrets/build.gradle.kts +++ b/airbyte-config/config-secrets/build.gradle.kts @@ -1,7 +1,7 @@ plugins { id("io.airbyte.gradle.jvm.lib") id("io.airbyte.gradle.publish") - id("java-test-fixtures") + `java-test-fixtures` kotlin("jvm") kotlin("kapt") } diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle deleted file mode 100644 index 645b0df8ea8..00000000000 --- a/airbyte-config/init/build.gradle +++ /dev/null @@ -1,51 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - implementation project(':airbyte-commons') - implementation 'commons-cli:commons-cli:1.4' - implementation project(':airbyte-config:specs') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-featureflag') - implementation project(':airbyte-notification') - implementation project(':airbyte-persistence:job-persistence') - implementation libs.airbyte.protocol - implementation project(':airbyte-json-validation') - compileOnly libs.lombok - annotationProcessor libs.lombok - implementation libs.guava - - testImplementation project(':airbyte-test-utils') - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - testImplementation libs.junit.pioneer - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok -} - -airbyte { - docker { - imageName = "init" - } -} - -def copyScripts = tasks.register("copyScripts", Copy) { - from('scripts') - into 'build/airbyte/docker/bin/scripts' -} - -tasks.named("dockerBuildImage") { - dependsOn copyScripts -} - -processResources { - from("${project.rootDir}/airbyte-connector-builder-resources") -} diff --git a/airbyte-config/init/build.gradle.kts b/airbyte-config/init/build.gradle.kts new file mode 100644 index 00000000000..fb04e27ae88 --- /dev/null +++ b/airbyte-config/init/build.gradle.kts @@ -0,0 +1,51 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(libs.bundles.micronaut.annotation.processor) + api(libs.bundles.micronaut.annotation) + + implementation(project(":airbyte-commons")) + implementation("commons-cli:commons-cli:1.4") + implementation(project(":airbyte-config:specs")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-featureflag")) + implementation(project(":airbyte-notification")) + implementation(project(":airbyte-persistence:job-persistence")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-json-validation")) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + implementation(libs.guava) + + testImplementation(project(":airbyte-test-utils")) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + testImplementation(libs.junit.pioneer) + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) +} + +airbyte { + docker { + imageName = "init" + } +} + +val copyScripts = tasks.register("copyScripts") { + from("scripts") + into("build/airbyte/docker/bin/scripts") +} + +tasks.named("dockerBuildImage") { + dependsOn(copyScripts) +} + +tasks.processResources { + from("${project.rootDir}/airbyte-connector-builder-resources") +} diff --git a/airbyte-config/specs/build.gradle b/airbyte-config/specs/build.gradle deleted file mode 100644 index 963c8048d79..00000000000 --- a/airbyte-config/specs/build.gradle +++ /dev/null @@ -1,49 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - id "de.undercouch.download" version "5.4.0" -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - implementation 'commons-cli:commons-cli:1.4' - implementation libs.commons.io - implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) - implementation libs.bundles.jackson - implementation libs.google.cloud.storage - implementation libs.micronaut.cache.caffeine - compileOnly libs.lombok - annotationProcessor libs.lombok - implementation project(':airbyte-commons') - implementation project(':airbyte-config:config-models') - implementation libs.airbyte.protocol - implementation project(':airbyte-json-validation') - implementation libs.okhttp - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1' - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - testImplementation libs.junit.pioneer -} - -airbyte { - spotless { - excludes = [ - "src/main/resources/seed/oss_registry.json", - "src/main/resources/seed/local_oss_registry.json" - ] - } -} - -def downloadConnectorRegistry = tasks.register("downloadConnectorRegistry", Download) { - src 'https://connectors.airbyte.com/files/registries/v0/oss_registry.json' - dest new File(projectDir, 'src/main/resources/seed/local_oss_registry.json') - overwrite true -} - -processResources.dependsOn(downloadConnectorRegistry) diff --git a/airbyte-config/specs/build.gradle.kts b/airbyte-config/specs/build.gradle.kts new file mode 100644 index 00000000000..d2c8e70bf5a --- /dev/null +++ b/airbyte-config/specs/build.gradle.kts @@ -0,0 +1,53 @@ +import de.undercouch.gradle.tasks.download.Download + +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + id("de.undercouch.download") version "5.4.0" +} + +dependencies { + annotationProcessor(libs.bundles.micronaut.annotation.processor) + api(libs.bundles.micronaut.annotation) + + implementation("commons-cli:commons-cli:1.4") + implementation(libs.commons.io) + implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) + implementation(libs.bundles.jackson) + implementation(libs.google.cloud.storage) + implementation(libs.micronaut.cache.caffeine) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-config:config-models")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-json-validation")) + implementation(libs.okhttp) + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + testImplementation("com.squareup.okhttp3:mockwebserver:4.9.1") + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + testImplementation(libs.junit.pioneer) +} + +airbyte { + spotless { + excludes = listOf( + "src/main/resources/seed/oss_registry.json", + "src/main/resources/seed/local_oss_registry.json", + ) + } +} + +val downloadConnectorRegistry = tasks.register("downloadConnectorRegistry") { + src("https://connectors.airbyte.com/files/registries/v0/oss_registry.json") + dest(File(projectDir, "src/main/resources/seed/local_oss_registry.json")) + overwrite( true) +} + +tasks.processResources { + dependsOn(downloadConnectorRegistry) +} diff --git a/airbyte-connector-builder-resources/CDK_VERSION b/airbyte-connector-builder-resources/CDK_VERSION index 4a1c22f3172..df174c967e9 100644 --- a/airbyte-connector-builder-resources/CDK_VERSION +++ b/airbyte-connector-builder-resources/CDK_VERSION @@ -1 +1 @@ -0.51.41 +0.55.2 diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index f0bf5e1473e..061ecb5a768 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_IMAGE=airbyte/airbyte-base-java-python-image:1.0 FROM ${BASE_IMAGE} AS connector-builder-server # Set up CDK requirements -ARG CDK_VERSION=0.51.41 +ARG CDK_VERSION=0.55.2 ENV CDK_PYTHON=${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin/python ENV CDK_ENTRYPOINT ${PYENV_ROOT}/versions/${PYTHON_VERSION}/lib/python3.9/site-packages/airbyte_cdk/connector_builder/main.py # Set up CDK diff --git a/airbyte-connector-builder-server/build.gradle b/airbyte-connector-builder-server/build.gradle deleted file mode 100644 index 58dfb5b46f2..00000000000 --- a/airbyte-connector-builder-server/build.gradle +++ /dev/null @@ -1,139 +0,0 @@ -import org.openapitools.generator.gradle.plugin.tasks.GenerateTask - -plugins { - id "io.airbyte.gradle.jvm.app" - id "io.airbyte.gradle.docker" - id "org.openapi.generator" - id "io.airbyte.gradle.publish" -} - -dependencies { - implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.1' - implementation 'com.googlecode.json-simple:json-simple:1.1.1' - - // Cloud service dependencies. These are not strictly necessary yet, but likely needed for any full-fledged cloud service - implementation libs.bundles.datadog - // implementation libs.bundles.temporal uncomment this when we start using temporal to invoke connector commands - implementation libs.sentry.java - - implementation libs.guava - - // Micronaut dependencies - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.micronaut.security - - implementation project(':airbyte-commons') - - // OpenAPI code generation dependencies - implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2' - - // Internal dependencies - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-protocol') - implementation project(':airbyte-commons-server') - implementation project(':airbyte-commons-worker') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-config:init') - implementation project(':airbyte-metrics:metrics-lib') - - implementation libs.airbyte.protocol - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer -} - -Properties env = new Properties() -rootProject.file('.env.dev').withInputStream { env.load(it) } - -airbyte { - application { - mainClass = 'io.airbyte.connector_builder.MicronautConnectorBuilderServerRunner' - defaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] - localEnvVars = env + [ - "AIRBYTE_ROLE" : System.getenv("AIRBYTE_ROLE") ?: "", - "AIRBYTE_VERSION": env.VERSION, - // path to CDK virtual environment - "CDK_PYTHON" : System.getenv('CDK_PYTHON') ?: "", - // path to CDK connector builder's main.py - "CDK_ENTRYPOINT" : System.getenv('CDK_ENTRYPOINT') ?: "" - ] as Map - } - docker { - imageName = "connector-builder-server" - } -} - -def generateOpenApiServer = tasks.register("generateOpenApiServer", GenerateTask) { - def generatedCodeDir = "$buildDir/generated/api/server" - def specFile = "$projectDir/src/main/openapi/openapi.yaml" - - inputs.file specFile - inputSpec = specFile - outputDir = generatedCodeDir - - generatorName = "jaxrs-spec" - apiPackage = "io.airbyte.connector_builder.api.generated" - invokerPackage = "io.airbyte.connector_builder.api.invoker.generated" - modelPackage = "io.airbyte.connector_builder.api.model.generated" - - schemaMappings = [ - 'ConnectorConfig' : 'com.fasterxml.jackson.databind.JsonNode', - 'ConnectorManifest': 'com.fasterxml.jackson.databind.JsonNode', - ] - - // Our spec does not have nullable, but if it changes, this would be a gotcha that we would want to avoid - configOptions = [ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly : "true", - /* - JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. - It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. - The below Jackson annotation is made to only keep non null values in serialized json. - We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. - Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. - */ - additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", - ] -} - -tasks.named("compileJava") { - dependsOn generateOpenApiServer -} - -// Ensures that the generated models are compiled during the build step so they are available for use at runtime -sourceSets { - main { - java { - srcDirs "$buildDir/generated/api/server/src/gen/java" - } - resources { - srcDir "$projectDir/src/main/openapi/" - } - } -} - -def copyPythonDeps = tasks.register("copyPythonDependencies", Copy) { - from "${project.projectDir}/requirements.txt" - into "build/airbyte/docker/" -} - -tasks.named("dockerBuildImage") { - // Set build args - // Current CDK version used by the Connector Builder and workers running Connector Builder connectors - String cdkVersion = new File( - project.projectDir.parentFile, - 'airbyte-connector-builder-resources/CDK_VERSION').text.trim() - buildArgs['CDK_VERSION'] = cdkVersion - - dependsOn copyPythonDeps - dependsOn generateOpenApiServer -} diff --git a/airbyte-connector-builder-server/build.gradle.kts b/airbyte-connector-builder-server/build.gradle.kts new file mode 100644 index 00000000000..3bbab49366f --- /dev/null +++ b/airbyte-connector-builder-server/build.gradle.kts @@ -0,0 +1,139 @@ +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask +import java.util.Properties + +plugins { + id("io.airbyte.gradle.jvm.app") + id("io.airbyte.gradle.docker") + id("org.openapi.generator") + id("io.airbyte.gradle.publish") +} + +dependencies { + implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.1") + implementation("com.googlecode.json-simple:json-simple:1.1.1") + + // Cloud service dependencies. These are not strictly necessary yet, but likely needed for any full-fledged cloud service) + implementation(libs.bundles.datadog) + // implementation(libs.bundles.temporal uncomment this when we start using temporal to invoke connector commands) + implementation(libs.sentry.java) + + implementation(libs.guava) + + // Micronaut dependencies) + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.micronaut.security) + + implementation(project(":airbyte-commons")) + + // OpenAPI code generation(dependencies) + implementation("io.swagger:swagger-annotations:1.6.2") + + // Internal dependencies) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-protocol")) + implementation(project(":airbyte-commons-server")) + implementation(project(":airbyte-commons-worker")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-config:init")) + implementation(project(":airbyte-metrics:metrics-lib")) + + implementation(libs.airbyte.protocol) + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) +} + +val env = Properties().apply { + load(rootProject.file(".env.dev").inputStream()) +} + +airbyte { + application { + mainClass = "io.airbyte.connector_builder.MicronautConnectorBuilderServerRunner" + defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + @Suppress("UNCHECKED_CAST") + localEnvVars.putAll(env.toMap() as Map) + localEnvVars.putAll(mapOf( + "AIRBYTE_ROLE" to (System.getenv("AIRBYTE_ROLE") ?: ""), + "AIRBYTE_VERSION" to env["VERSION"].toString(), + // path to CDK virtual environment) + "CDK_PYTHON" to (System.getenv("CDK_PYTHON") ?: ""), + // path to CDK connector builder"s main.py) + "CDK_ENTRYPOINT" to (System.getenv("CDK_ENTRYPOINT") ?: ""), + )) + } + docker { + imageName = "connector-builder-server" + } +} + +val generateOpenApiServer = tasks.register("generateOpenApiServer") { + val specFile = "$projectDir/src/main/openapi/openapi.yaml" + inputs.file(specFile) + inputSpec = specFile + outputDir = "$buildDir/generated/api/server" + + generatorName = "jaxrs-spec" + apiPackage = "io.airbyte.connector_builder.api.generated" + invokerPackage = "io.airbyte.connector_builder.api.invoker.generated" + modelPackage = "io.airbyte.connector_builder.api.model.generated" + + schemaMappings.putAll(mapOf( + "ConnectorConfig" to "com.fasterxml.jackson.databind.JsonNode", + "ConnectorManifest" to "com.fasterxml.jackson.databind.JsonNode", + )) + + // Our spec does not have nullable, but if it changes, this would be a gotcha that we would want to avoid) + configOptions.putAll(mapOf( + "dateLibrary" to "java8", + "generatePom" to "false", + "interfaceOnly" to "true", + /*) + JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. + It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. + The below Jackson annotation(is made to only(keep non null values in serialized json. + We are not yet using nullable=true properties in our OpenApi so this is a valid(workaround at the moment to circumvent the default JAX-RS behavior described above. + Feel free to read the conversation(on https://github.com/airbytehq/airbyte/pull/13370 for more details. + */ + "additionalModelTypeAnnotations" to "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", + )) +} + +tasks.named("compileJava") { + dependsOn(generateOpenApiServer) +} +//// Ensures that the generated models are compiled during the build step so they are available for use at runtime) + +sourceSets { + main { + java { + srcDirs("$buildDir/generated/api/server/src/gen/java") + } + resources { + srcDir("$projectDir/src/main/openapi/") + } + } +} + +val copyPythonDeps = tasks.register("copyPythonDependencies") { + from("$projectDir/requirements.txt") + into("$buildDir/airbyte/docker/") +} +// +tasks.named("dockerBuildImage") { + // Set build args + // Current CDK version(used by the Connector Builder and workers running Connector Builder connectors + val cdkVersion: String = File(project.projectDir.parentFile, "airbyte-connector-builder-resources/CDK_VERSION").readText().trim() + buildArgs.put("CDK_VERSION", cdkVersion) + + dependsOn(copyPythonDeps, generateOpenApiServer) +} diff --git a/airbyte-connector-builder-server/requirements.in b/airbyte-connector-builder-server/requirements.in index 9fe642b9fa0..37afe97b6b6 100644 --- a/airbyte-connector-builder-server/requirements.in +++ b/airbyte-connector-builder-server/requirements.in @@ -1 +1 @@ -airbyte-cdk==0.51.41 +airbyte-cdk==0.55.2 diff --git a/airbyte-connector-builder-server/requirements.txt b/airbyte-connector-builder-server/requirements.txt index 9ac14be8ffd..5a01069ef9e 100644 --- a/airbyte-connector-builder-server/requirements.txt +++ b/airbyte-connector-builder-server/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile # -airbyte-cdk==0.51.41 +airbyte-cdk==0.55.2 # via -r requirements.in airbyte-protocol-models==0.4.2 # via airbyte-cdk @@ -17,23 +17,23 @@ backoff==2.2.1 # via airbyte-cdk bracex==2.4 # via wcmatch -cachetools==5.3.1 +cachetools==5.3.2 # via airbyte-cdk -cattrs==23.1.2 +cattrs==23.2.2 # via requests-cache -certifi==2023.7.22 +certifi==2023.11.17 # via requests -charset-normalizer==3.3.0 +charset-normalizer==3.3.2 # via requests deprecated==1.2.14 # via airbyte-cdk dpath==2.0.8 # via airbyte-cdk -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via cattrs genson==1.2.2 # via airbyte-cdk -idna==3.4 +idna==3.6 # via requests isodate==0.6.1 # via airbyte-cdk @@ -47,13 +47,15 @@ markupsafe==2.1.3 # via jinja2 pendulum==2.1.2 # via airbyte-cdk -platformdirs==3.11.0 +platformdirs==4.0.0 # via requests-cache pydantic==1.10.13 # via # airbyte-cdk # airbyte-protocol-models -pyrsistent==0.19.3 +pyrate-limiter==3.1.0 + # via airbyte-cdk +pyrsistent==0.20.0 # via jsonschema python-dateutil==2.8.2 # via @@ -67,7 +69,7 @@ requests==2.31.0 # via # airbyte-cdk # requests-cache -requests-cache==1.1.0 +requests-cache==1.1.1 # via airbyte-cdk six==1.16.0 # via @@ -81,13 +83,13 @@ typing-extensions==4.8.0 # pydantic url-normalize==1.4.3 # via requests-cache -urllib3==2.0.7 +urllib3==2.1.0 # via # requests # requests-cache wcmatch==8.4 # via airbyte-cdk -wrapt==1.15.0 +wrapt==1.16.0 # via deprecated # The following packages are considered to be unsafe in a requirements file: diff --git a/airbyte-container-orchestrator/build.gradle b/airbyte-container-orchestrator/build.gradle deleted file mode 100644 index d7ad969bc24..00000000000 --- a/airbyte-container-orchestrator/build.gradle +++ /dev/null @@ -1,104 +0,0 @@ -import groovy.json.JsonBuilder -import groovy.yaml.YamlSlurper - -import java.util.zip.ZipFile - -plugins { - id "io.airbyte.gradle.jvm.app" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -configurations { - airbyteProtocol -} - -configurations.all { - resolutionStrategy { - // Ensure that the versions defined in deps.toml are used - // instead of versions from transitive dependencies - // Force to avoid updated version brought in transitively from Micronaut 3.8+ - // that is incompatible with our current Helm setup - force libs.s3, libs.aws.java.sdk.s3 - } -} -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - annotationProcessor libs.lombok - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.guava - implementation libs.s3 - implementation libs.aws.java.sdk.s3 - implementation libs.sts - implementation libs.kubernetes.client - implementation libs.bundles.datadog - implementation libs.bundles.log4j - compileOnly libs.lombok - - implementation project(':airbyte-api') - implementation project(':airbyte-commons') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-commons-converters') - implementation project(':airbyte-commons-protocol') - implementation project(':airbyte-commons-micronaut') - implementation project(':airbyte-commons-temporal') - implementation project(':airbyte-commons-with-dependencies') - implementation project(':airbyte-commons-worker') - implementation project(':airbyte-config:init') - implementation project(':airbyte-featureflag') - implementation project(':airbyte-json-validation') - implementation libs.airbyte.protocol - implementation project(':airbyte-metrics:metrics-lib') - implementation project(':airbyte-worker-models') - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - testImplementation libs.bundles.micronaut.test - testImplementation libs.mockito.inline - testImplementation libs.postgresql - testImplementation libs.platform.testcontainers - testImplementation libs.platform.testcontainers.postgresql - testImplementation libs.bundles.bouncycastle - - airbyteProtocol(libs.airbyte.protocol) { - transitive = false - } -} - -airbyte { - application { - mainClass = "io.airbyte.container_orchestrator.Application" - defaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] - } - docker { - imageName = "container-orchestrator" - } -} - -// Duplicated from :airbyte-worker, eventually, this should be handled in :airbyte-protocol -def generateWellKnownTypes = tasks.register("generateWellKnownTypes") { - inputs.files(configurations.airbyteProtocol) // declaring inputs - def targetFile = project.file("build/airbyte/docker/WellKnownTypes.json") - outputs.file(targetFile) // declaring outputs - doLast { - def wellKnownTypesYamlPath = 'airbyte_protocol/well_known_types.yaml' - configurations.airbyteProtocol.getFiles().each { - def zip = new ZipFile(it) - def entry = zip.getEntry(wellKnownTypesYamlPath) - - def wellKnownTypesYaml = zip.getInputStream(entry).text - def parsedYaml = new YamlSlurper().parseText(wellKnownTypesYaml) - def wellKnownTypesJson = new JsonBuilder(parsedYaml).toPrettyString() - targetFile.getParentFile().mkdirs() - targetFile.text = wellKnownTypesJson - } - } -} - -tasks.named("dockerBuildImage") { - dependsOn generateWellKnownTypes -} diff --git a/airbyte-container-orchestrator/build.gradle.kts b/airbyte-container-orchestrator/build.gradle.kts new file mode 100644 index 00000000000..e2724a8e42f --- /dev/null +++ b/airbyte-container-orchestrator/build.gradle.kts @@ -0,0 +1,121 @@ +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule + +import java.util.zip.ZipFile + +buildscript { + repositories { + mavenCentral() + } + dependencies { + // necessary to convert the well_know_types from yaml to json + val jacksonVersion = "2.16.0" + classpath("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$jacksonVersion") + classpath("com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion") + } +} + +plugins { + id("io.airbyte.gradle.jvm.app") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +val airbyteProtocol by configurations.creating + +configurations.all { + resolutionStrategy { + // Ensure that the versions defined in deps.toml are used) + // instead of versions from transitive dependencies) + // Force to avoid(updated version brought in transitively from Micronaut 3.8+) + // that is incompatible with our current Helm setup) + force (libs.s3, libs.aws.java.sdk.s3) + } +} +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + annotationProcessor(libs.lombok) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.guava) + implementation(libs.s3) + implementation(libs.aws.java.sdk.s3) + implementation(libs.sts) + implementation(libs.kubernetes.client) + implementation(libs.bundles.datadog) + implementation(libs.bundles.log4j) + compileOnly(libs.lombok) + + implementation(project(":airbyte-api")) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-commons-converters")) + implementation(project(":airbyte-commons-protocol")) + implementation(project(":airbyte-commons-micronaut")) + implementation(project(":airbyte-commons-temporal")) + implementation(project(":airbyte-commons-with-dependencies")) + implementation(project(":airbyte-commons-worker")) + implementation(project(":airbyte-config:init")) + implementation(project(":airbyte-featureflag")) + implementation(project(":airbyte-json-validation")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-metrics:metrics-lib")) + implementation(project(":airbyte-worker-models")) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.mockito.inline) + testImplementation(libs.postgresql) + testImplementation(libs.platform.testcontainers) + testImplementation(libs.platform.testcontainers.postgresql) + testImplementation(libs.bundles.bouncycastle) + + airbyteProtocol(libs.airbyte.protocol) { + isTransitive = false + } +} + +airbyte { + application { + mainClass = "io.airbyte.container_orchestrator.Application" + defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + } + docker { + imageName = "container-orchestrator" + } +} + +// Duplicated from :airbyte-worker, eventually, this should be handled in :airbyte-protocol) +val generateWellKnownTypes = tasks.register("generateWellKnownTypes") { + inputs.files(airbyteProtocol) // declaring inputs) + val targetFile = project.file("build/airbyte/docker/WellKnownTypes.json") + outputs.file(targetFile) // declaring outputs) + + doLast { + val wellKnownTypesYamlPath = "airbyte_protocol/well_known_types.yaml" + airbyteProtocol.files.forEach { + val zip = ZipFile(it) + val entry = zip.getEntry(wellKnownTypesYamlPath) + + val wellKnownTypesYaml = zip.getInputStream(entry).bufferedReader().use { reader -> reader.readText() } + val rawJson = yamlToJson(wellKnownTypesYaml) + targetFile.getParentFile().mkdirs() + targetFile.writeText(rawJson) + } + } +} + +tasks.named("dockerBuildImage") { + dependsOn(generateWellKnownTypes) +} + +fun yamlToJson(rawYaml: String): String { + val mappedYaml: Any = YAMLMapper().registerKotlinModule().readValue(rawYaml) + return ObjectMapper().registerKotlinModule().writeValueAsString(mappedYaml) +} diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/EventListeners.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/EventListeners.java index a4bddec97aa..d1f8111f575 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/EventListeners.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/EventListeners.java @@ -64,7 +64,7 @@ public class EventListeners { */ @EventListener void setEnvVars(final ServerStartupEvent unused) { - log.info("settings env vars"); + log.debug("settings env vars"); OrchestratorConstants.ENV_VARS_TO_TRANSFER.stream() .filter(envVars::containsKey) @@ -78,7 +78,7 @@ void setEnvVars(final ServerStartupEvent unused) { */ @EventListener void setLogging(final ServerStartupEvent unused) { - log.info("started logging"); + log.debug("started logging"); // make sure the new configuration is picked up final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); diff --git a/airbyte-cron/build.gradle b/airbyte-cron/build.gradle deleted file mode 100644 index b5609c9ad95..00000000000 --- a/airbyte-cron/build.gradle +++ /dev/null @@ -1,73 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.app" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" - id "org.jetbrains.kotlin.jvm" - id "org.jetbrains.kotlin.kapt" -} - -dependencies { - annotationProcessor libs.lombok - implementation libs.lombok - implementation libs.commons.io - - implementation libs.bundles.kubernetes.client - implementation libs.bundles.temporal - implementation libs.bundles.datadog - implementation libs.failsafe - implementation libs.failsafe.okhttp - implementation libs.java.jwt - implementation libs.kotlin.logging - implementation libs.okhttp - implementation libs.sentry.java - - implementation project(':airbyte-api') - implementation project(':airbyte-analytics') - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-auth') - implementation project(':airbyte-commons-micronaut') - implementation project(':airbyte-commons-temporal') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-config:init') - implementation project(':airbyte-json-validation') - implementation project(':airbyte-data') - implementation project(':airbyte-db:db-lib') - implementation project(":airbyte-featureflag") - implementation project(':airbyte-metrics:metrics-lib') - implementation project(':airbyte-persistence:job-persistence') - - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - kapt libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut -} - -Properties env = new Properties() -rootProject.file('.env.dev').withInputStream { env.load(it) } - -airbyte { - application { - mainClass = 'io.airbyte.cron.MicronautCronRunner' - defaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] - localEnvVars = env + [ - "AIRBYTE_ROLE" : System.getenv("AIRBYTE_ROLE") ?: "undefined", - "AIRBYTE_VERSION": env.VERSION - ] as Map - } - - docker { - imageName = "cron" - } -} - -// The DuplicatesStrategy will be required while this module is mixture of kotlin and java _with_ lombok dependencies. -// Kapt, by default, runs all annotation processors and disables annotation processing by javac, however -// this default behavior breaks the lombok java annotation processor. To avoid lombok breaking, kapt has -// keepJavacAnnotationProcessors enabled, which causes duplicate META-INF files to be generated. -// Once lombok has been removed, this can also be removed. -tasks.withType(Jar).configureEach { - duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} diff --git a/airbyte-cron/build.gradle.kts b/airbyte-cron/build.gradle.kts new file mode 100644 index 00000000000..eece4a3ade1 --- /dev/null +++ b/airbyte-cron/build.gradle.kts @@ -0,0 +1,81 @@ +import java.util.Properties + +plugins { + id("io.airbyte.gradle.jvm.app") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") + kotlin("jvm") + kotlin("kapt") +} + +dependencies { + annotationProcessor(libs.lombok) + implementation(libs.lombok) + implementation(libs.commons.io) + + implementation(libs.bundles.kubernetes.client) + implementation(libs.bundles.temporal) + implementation(libs.bundles.datadog) + implementation(libs.failsafe) + implementation(libs.failsafe.okhttp) + implementation(libs.java.jwt) + implementation(libs.kotlin.logging) + implementation(libs.okhttp) + implementation(libs.sentry.java) + + implementation(project(":airbyte-api")) + implementation(project(":airbyte-analytics")) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-auth")) + implementation(project(":airbyte-commons-micronaut")) + implementation(project(":airbyte-commons-temporal")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-config:init")) + implementation(project(":airbyte-json-validation")) + implementation(project(":airbyte-data")) + implementation(project(":airbyte-db:db-lib")) + implementation(project(":airbyte-featureflag")) + implementation(project(":airbyte-metrics:metrics-lib")) + implementation(project(":airbyte-persistence:job-persistence")) + + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + kapt(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + + testImplementation(libs.bundles.junit) + testImplementation(libs.mockk) +} + +val env = Properties().apply { + load(rootProject.file(".env.dev").inputStream()) +} + +airbyte { + application { + mainClass = "io.airbyte.cron.MicronautCronRunner" + defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + @Suppress("UNCHECKED_CAST") + localEnvVars.putAll(env.toMap() as Map) + localEnvVars.putAll(mapOf( + "AIRBYTE_ROLE" to (System.getenv("AIRBYTE_ROLE") ?: "undefined"), + "AIRBYTE_VERSION" to env["VERSION"].toString(), + )) + } + + docker { + imageName = "cron" + } +} + +// The DuplicatesStrategy will be required while this module is mixture of kotlin and java _with_ lombok dependencies.) +// Kapt, by default, runs all annotation(processors and disables annotation(processing by javac, however) +// this default behavior breaks the lombok java annotation(processor. To avoid(lombok breaking, kapt has) +// keepJavacAnnotationProcessors enabled, which causes duplicate META-INF files to be generated.) +// Once lombok has been removed, this can also be removed.) +tasks.withType().configureEach { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/jobs/WorkloadMonitor.kt b/airbyte-cron/src/main/java/io/airbyte/cron/jobs/WorkloadMonitor.kt index d32f4ca0118..346706a2f5c 100644 --- a/airbyte-cron/src/main/java/io/airbyte/cron/jobs/WorkloadMonitor.kt +++ b/airbyte-cron/src/main/java/io/airbyte/cron/jobs/WorkloadMonitor.kt @@ -1,5 +1,10 @@ package io.airbyte.cron.jobs +import datadog.trace.api.Trace +import io.airbyte.metrics.lib.MetricAttribute +import io.airbyte.metrics.lib.MetricClient +import io.airbyte.metrics.lib.MetricTags +import io.airbyte.metrics.lib.OssMetricsRegistry import io.airbyte.workload.api.client.generated.WorkloadApi import io.airbyte.workload.api.client.model.generated.Workload import io.airbyte.workload.api.client.model.generated.WorkloadCancelRequest @@ -13,6 +18,7 @@ import jakarta.inject.Named import jakarta.inject.Singleton import java.time.Duration import java.time.OffsetDateTime +import java.time.ZoneId import java.time.ZoneOffset private val logger = KotlinLogging.logger { } @@ -27,11 +33,14 @@ class WorkloadMonitor( @Property(name = "airbyte.workload.monitor.claim-timeout") private val claimTimeout: Duration, @Property(name = "airbyte.workload.monitor.heartbeat-timeout") private val heartbeatTimeout: Duration, @Named("replicationNotStartedTimeout") private val nonStartedTimeout: Duration, + private val metricClient: MetricClient, + private val timeProvider: (ZoneId) -> OffsetDateTime = OffsetDateTime::now, ) { + @Trace @Scheduled(fixedRate = "\${airbyte.workload.monitor.not-started-check-rate}") fun cancelNotStartedWorkloads() { logger.info { "Checking for not started workloads." } - val oldestStartedTime = OffsetDateTime.now(ZoneOffset.UTC).minusSeconds(nonStartedTimeout.seconds) + val oldestStartedTime = timeProvider(ZoneOffset.UTC).minusSeconds(nonStartedTimeout.seconds) val notStartedWorkloads = workloadApi.workloadList( WorkloadListRequest( @@ -40,13 +49,14 @@ class WorkloadMonitor( ), ) - cancelWorkloads(notStartedWorkloads.workloads, "Not started within time limit") + cancelWorkloads(notStartedWorkloads.workloads, "Not started within time limit", "workload-monitor-start") } + @Trace @Scheduled(fixedRate = "\${airbyte.workload.monitor.claim-check-rate}") fun cancelNotClaimedWorkloads() { logger.info { "Checking for not claimed workloads." } - val oldestClaimTime = OffsetDateTime.now(ZoneOffset.UTC).minusSeconds(claimTimeout.seconds) + val oldestClaimTime = timeProvider(ZoneOffset.UTC).minusSeconds(claimTimeout.seconds) val notClaimedWorkloads = workloadApi.workloadList( WorkloadListRequest( @@ -55,13 +65,14 @@ class WorkloadMonitor( ), ) - cancelWorkloads(notClaimedWorkloads.workloads, "Not claimed within time limit") + cancelWorkloads(notClaimedWorkloads.workloads, "Not claimed within time limit", "workload-monitor-claim") } + @Trace @Scheduled(fixedRate = "\${airbyte.workload.monitor.heartbeat-check-rate}") fun cancelNotHeartbeatingWorkloads() { logger.info { "Checking for non heartbeating workloads." } - val oldestHeartbeatTime = OffsetDateTime.now(ZoneOffset.UTC).minusSeconds(heartbeatTimeout.seconds) + val oldestHeartbeatTime = timeProvider(ZoneOffset.UTC).minusSeconds(heartbeatTimeout.seconds) val nonHeartbeatingWorkloads = workloadApi.workloadList( WorkloadListRequest( @@ -70,19 +81,29 @@ class WorkloadMonitor( ), ) - cancelWorkloads(nonHeartbeatingWorkloads.workloads, "No heartbeat within time limit") + cancelWorkloads(nonHeartbeatingWorkloads.workloads, "No heartbeat within time limit", "workload-monitor-heartbeat") } private fun cancelWorkloads( workloads: List, reason: String, + source: String, ) { workloads.map { + var status = "fail" try { logger.info { "Cancelling workload ${it.id}, reason: $reason" } - workloadApi.workloadCancel(WorkloadCancelRequest(workloadId = it.id, reason = reason, source = "workload-monitor")) + workloadApi.workloadCancel(WorkloadCancelRequest(workloadId = it.id, reason = reason, source = source)) + status = "ok" } catch (e: Exception) { logger.warn(e) { "Failed to cancel workload ${it.id}" } + } finally { + metricClient.count( + OssMetricsRegistry.WORKLOADS_CANCEL, + 1, + MetricAttribute(MetricTags.CANCELLATION_SOURCE, source), + MetricAttribute(MetricTags.STATUS, status), + ) } } } diff --git a/airbyte-cron/src/test/kotlin/io/airbyte/cron/jobs/WorkloadMonitorTest.kt b/airbyte-cron/src/test/kotlin/io/airbyte/cron/jobs/WorkloadMonitorTest.kt new file mode 100644 index 00000000000..6d3318966c2 --- /dev/null +++ b/airbyte-cron/src/test/kotlin/io/airbyte/cron/jobs/WorkloadMonitorTest.kt @@ -0,0 +1,168 @@ +package io.airbyte.cron.jobs + +import io.airbyte.metrics.lib.MetricAttribute +import io.airbyte.metrics.lib.MetricClient +import io.airbyte.metrics.lib.MetricTags +import io.airbyte.metrics.lib.OssMetricsRegistry +import io.airbyte.workload.api.client.generated.WorkloadApi +import io.airbyte.workload.api.client.model.generated.Workload +import io.airbyte.workload.api.client.model.generated.WorkloadListResponse +import io.airbyte.workload.api.client.model.generated.WorkloadStatus +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkClass +import io.mockk.verify +import io.mockk.verifyAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.openapitools.client.infrastructure.ServerException +import java.time.Duration +import java.time.OffsetDateTime +import java.time.ZoneId +import java.time.temporal.ChronoUnit + +class WorkloadMonitorTest { + val claimTimeout = Duration.of(5, ChronoUnit.SECONDS) + val heartbeatTimeout = Duration.of(6, ChronoUnit.SECONDS) + val nonStartedTimeout = Duration.of(7, ChronoUnit.SECONDS) + + lateinit var currentTime: OffsetDateTime + lateinit var metricClient: MetricClient + lateinit var workloadApi: WorkloadApi + lateinit var workloadMonitor: WorkloadMonitor + + @BeforeEach + fun beforeEach() { + metricClient = + mockk().also { + every { it.count(any(), any(), *anyVararg()) } returns Unit + } + workloadApi = mockk() + workloadMonitor = + WorkloadMonitor( + workloadApi = workloadApi, + claimTimeout = claimTimeout, + heartbeatTimeout = heartbeatTimeout, + nonStartedTimeout = nonStartedTimeout, + metricClient = metricClient, + timeProvider = { _: ZoneId -> currentTime }, + ) + } + + @Test + fun `test cancel not started workloads`() { + val expiredWorkloads = WorkloadListResponse(workloads = listOf(getWorkload("1"), getWorkload("2"), getWorkload("3"))) + currentTime = OffsetDateTime.now() + every { workloadApi.workloadList(any()) } returns expiredWorkloads + every { workloadApi.workloadCancel(any()) } returns Unit andThenThrows ServerException() andThen Unit + + workloadMonitor.cancelNotStartedWorkloads() + + verifyAll { + workloadApi.workloadList( + match { + it.status == listOf(WorkloadStatus.CLAIMED) && it.updatedBefore == currentTime.minus(nonStartedTimeout) + }, + ) + workloadApi.workloadCancel(match { it.workloadId == "1" }) + workloadApi.workloadCancel(match { it.workloadId == "2" }) + workloadApi.workloadCancel(match { it.workloadId == "3" }) + } + verify(exactly = 2) { + metricClient.count( + OssMetricsRegistry.WORKLOADS_CANCEL, + 1, + MetricAttribute(MetricTags.CANCELLATION_SOURCE, "workload-monitor-start"), + MetricAttribute(MetricTags.STATUS, "ok"), + ) + } + verify(exactly = 1) { + metricClient.count( + OssMetricsRegistry.WORKLOADS_CANCEL, + 1, + MetricAttribute(MetricTags.CANCELLATION_SOURCE, "workload-monitor-start"), + MetricAttribute(MetricTags.STATUS, "fail"), + ) + } + } + + @Test + fun `test cancel not claimed workloads`() { + val expiredWorkloads = WorkloadListResponse(workloads = listOf(getWorkload("a"), getWorkload("b"), getWorkload("c"))) + currentTime = OffsetDateTime.now() + every { workloadApi.workloadList(any()) } returns expiredWorkloads + every { workloadApi.workloadCancel(any()) } throws ServerException() andThen Unit andThen Unit + + workloadMonitor.cancelNotClaimedWorkloads() + + verifyAll { + workloadApi.workloadList( + match { + it.status == listOf(WorkloadStatus.PENDING) && it.updatedBefore == currentTime.minus(claimTimeout) + }, + ) + workloadApi.workloadCancel(match { it.workloadId == "a" }) + workloadApi.workloadCancel(match { it.workloadId == "b" }) + workloadApi.workloadCancel(match { it.workloadId == "c" }) + } + verify(exactly = 2) { + metricClient.count( + OssMetricsRegistry.WORKLOADS_CANCEL, + 1, + MetricAttribute(MetricTags.CANCELLATION_SOURCE, "workload-monitor-claim"), + MetricAttribute(MetricTags.STATUS, "ok"), + ) + } + verify(exactly = 1) { + metricClient.count( + OssMetricsRegistry.WORKLOADS_CANCEL, + 1, + MetricAttribute(MetricTags.CANCELLATION_SOURCE, "workload-monitor-claim"), + MetricAttribute(MetricTags.STATUS, "fail"), + ) + } + } + + @Test + fun `test cancel not heartbeating workloads`() { + val expiredWorkloads = WorkloadListResponse(workloads = listOf(getWorkload("3"), getWorkload("4"), getWorkload("5"))) + currentTime = OffsetDateTime.now() + every { workloadApi.workloadList(any()) } returns expiredWorkloads + every { workloadApi.workloadCancel(any()) } returns Unit andThenThrows ServerException() andThen Unit + + workloadMonitor.cancelNotHeartbeatingWorkloads() + + verifyAll { + workloadApi.workloadList( + match { + it.status == listOf(WorkloadStatus.RUNNING) && it.updatedBefore == currentTime.minus(heartbeatTimeout) + }, + ) + workloadApi.workloadCancel(match { it.workloadId == "3" }) + workloadApi.workloadCancel(match { it.workloadId == "4" }) + workloadApi.workloadCancel(match { it.workloadId == "5" }) + } + verify(exactly = 2) { + metricClient.count( + OssMetricsRegistry.WORKLOADS_CANCEL, + 1, + MetricAttribute(MetricTags.CANCELLATION_SOURCE, "workload-monitor-heartbeat"), + MetricAttribute(MetricTags.STATUS, "ok"), + ) + } + verify(exactly = 1) { + metricClient.count( + OssMetricsRegistry.WORKLOADS_CANCEL, + 1, + MetricAttribute(MetricTags.CANCELLATION_SOURCE, "workload-monitor-heartbeat"), + MetricAttribute(MetricTags.STATUS, "fail"), + ) + } + } + + fun getWorkload(id: String): Workload { + return mockkClass(Workload::class).also { + every { it.id } returns id + } + } +} diff --git a/airbyte-data/build.gradle b/airbyte-data/build.gradle deleted file mode 100644 index 32f2ad58f26..00000000000 --- a/airbyte-data/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - implementation libs.bundles.apache - implementation libs.bundles.jackson - implementation libs.guava - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-protocol') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-secrets') - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-db:jooq') - implementation project(':airbyte-json-validation') - implementation project(':airbyte-featureflag') - implementation libs.airbyte.protocol - - compileOnly libs.lombok - annotationProcessor libs.lombok -} diff --git a/airbyte-data/build.gradle.kts b/airbyte-data/build.gradle.kts new file mode 100644 index 00000000000..c0a3f7537b1 --- /dev/null +++ b/airbyte-data/build.gradle.kts @@ -0,0 +1,25 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(libs.bundles.micronaut.annotation.processor) + api(libs.bundles.micronaut.annotation) + + implementation(libs.bundles.apache) + implementation(libs.bundles.jackson) + implementation(libs.guava) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-protocol")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-secrets")) + implementation(project(":airbyte-db:db-lib")) + implementation(project(":airbyte-db:jooq")) + implementation(project(":airbyte-json-validation")) + implementation(project(":airbyte-featureflag")) + implementation(libs.airbyte.protocol) + + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/ApiKeyService.java b/airbyte-data/src/main/java/io/airbyte/data/services/ApiKeyService.java new file mode 100644 index 00000000000..08e78301089 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/ApiKeyService.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.config.ApiKey; +import io.airbyte.config.User; +import java.util.List; +import java.util.Optional; + +public interface ApiKeyService { + + ApiKey createApiKeyForUser(User userId); + + List listApiKeysForUser(User userId); + + Optional deleteApiKey(String apiKeyId); + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImpl.java b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImpl.java index 231df09a2c6..4f38e6be1c4 100644 --- a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImpl.java +++ b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImpl.java @@ -13,7 +13,6 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.enums.Enums; -import io.airbyte.commons.json.Jsons; import io.airbyte.commons.version.AirbyteProtocolVersion; import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionBreakingChange; @@ -27,18 +26,14 @@ import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; import io.airbyte.db.instance.configs.jooq.generated.Tables; -import io.airbyte.db.instance.configs.jooq.generated.enums.ReleaseStage; -import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; import io.airbyte.db.instance.configs.jooq.generated.tables.records.ActorDefinitionWorkspaceGrantRecord; import jakarta.inject.Named; import jakarta.inject.Singleton; import java.io.IOException; -import java.time.LocalDate; import java.time.OffsetDateTime; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -47,7 +42,6 @@ import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.InsertSetMoreStep; -import org.jooq.JSONB; import org.jooq.JoinType; import org.jooq.Record; import org.jooq.Record1; @@ -513,52 +507,7 @@ private Optional getDefaultVersionForActorDefinitionIdOp * field from the DB. */ private ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionVersion actorDefinitionVersion, final DSLContext ctx) { - final OffsetDateTime timestamp = OffsetDateTime.now(); - // Generate a new UUID if one is not provided. Passing an ID is useful for mocks. - final UUID versionId = actorDefinitionVersion.getVersionId() != null ? actorDefinitionVersion.getVersionId() : UUID.randomUUID(); - - ctx.insertInto(Tables.ACTOR_DEFINITION_VERSION) - .set(Tables.ACTOR_DEFINITION_VERSION.ID, versionId) - .set(ACTOR_DEFINITION_VERSION.CREATED_AT, timestamp) - .set(ACTOR_DEFINITION_VERSION.UPDATED_AT, timestamp) - .set(Tables.ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, actorDefinitionVersion.getActorDefinitionId()) - .set(Tables.ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, actorDefinitionVersion.getDockerRepository()) - .set(Tables.ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, actorDefinitionVersion.getDockerImageTag()) - .set(Tables.ACTOR_DEFINITION_VERSION.SPEC, JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSpec()))) - .set(Tables.ACTOR_DEFINITION_VERSION.DOCUMENTATION_URL, actorDefinitionVersion.getDocumentationUrl()) - .set(Tables.ACTOR_DEFINITION_VERSION.PROTOCOL_VERSION, actorDefinitionVersion.getProtocolVersion()) - .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL, actorDefinitionVersion.getSupportLevel() == null ? null - : Enums.toEnum(actorDefinitionVersion.getSupportLevel().value(), - SupportLevel.class).orElseThrow()) - .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_STAGE, actorDefinitionVersion.getReleaseStage() == null ? null - : Enums.toEnum(actorDefinitionVersion.getReleaseStage().value(), - ReleaseStage.class).orElseThrow()) - .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_DATE, actorDefinitionVersion.getReleaseDate() == null ? null - : LocalDate.parse(actorDefinitionVersion.getReleaseDate())) - .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_REPOSITORY, - Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) - ? actorDefinitionVersion.getNormalizationConfig().getNormalizationRepository() - : null) - .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_TAG, - Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) - ? actorDefinitionVersion.getNormalizationConfig().getNormalizationTag() - : null) - .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORTS_DBT, actorDefinitionVersion.getSupportsDbt()) - .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_INTEGRATION_TYPE, - Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) - ? actorDefinitionVersion.getNormalizationConfig().getNormalizationIntegrationType() - : null) - .set(Tables.ACTOR_DEFINITION_VERSION.ALLOWED_HOSTS, actorDefinitionVersion.getAllowedHosts() == null ? null - : JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getAllowedHosts()))) - .set(Tables.ACTOR_DEFINITION_VERSION.SUGGESTED_STREAMS, - actorDefinitionVersion.getSuggestedStreams() == null ? null - : JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSuggestedStreams()))) - .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_STATE, - Enums.toEnum(actorDefinitionVersion.getSupportState().value(), io.airbyte.db.instance.configs.jooq.generated.enums.SupportState.class) - .orElseThrow()) - .execute(); - - return actorDefinitionVersion.withVersionId(versionId); + return ActorDefinitionVersionJooqHelper.writeActorDefinitionVersion(actorDefinitionVersion, ctx); } /** diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionVersionJooqHelper.java b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionVersionJooqHelper.java new file mode 100644 index 00000000000..29e800b8a29 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionVersionJooqHelper.java @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services.impls.jooq; + +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION_VERSION; + +import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.version.Version; +import io.airbyte.config.ActorDefinitionBreakingChange; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.db.instance.configs.jooq.generated.Tables; +import io.airbyte.db.instance.configs.jooq.generated.enums.ReleaseStage; +import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; +import java.io.IOException; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import org.jooq.DSLContext; +import org.jooq.JSONB; + +/** + * Helper class to keep shared jooq logic used in actors (source, destination). This should be + * removed in favor of a combined Actor service that deals with both sources and destinations + * uniformly. + */ +public class ActorDefinitionVersionJooqHelper { + + /** + * Insert an actor definition version. + * + * @param actorDefinitionVersion - actor definition version to insert + * @param ctx database context + * @throws IOException - you never know when you io + * @returns the POJO associated with the actor definition version inserted. Contains the versionId + * field from the DB. + */ + public static ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionVersion actorDefinitionVersion, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + // Generate a new UUID if one is not provided. Passing an ID is useful for mocks. + final UUID versionId = actorDefinitionVersion.getVersionId() != null ? actorDefinitionVersion.getVersionId() : UUID.randomUUID(); + + ctx.insertInto(Tables.ACTOR_DEFINITION_VERSION) + .set(Tables.ACTOR_DEFINITION_VERSION.ID, versionId) + .set(ACTOR_DEFINITION_VERSION.CREATED_AT, timestamp) + .set(ACTOR_DEFINITION_VERSION.UPDATED_AT, timestamp) + .set(Tables.ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, actorDefinitionVersion.getActorDefinitionId()) + .set(Tables.ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, actorDefinitionVersion.getDockerRepository()) + .set(Tables.ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, actorDefinitionVersion.getDockerImageTag()) + .set(Tables.ACTOR_DEFINITION_VERSION.SPEC, JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSpec()))) + .set(Tables.ACTOR_DEFINITION_VERSION.DOCUMENTATION_URL, actorDefinitionVersion.getDocumentationUrl()) + .set(Tables.ACTOR_DEFINITION_VERSION.PROTOCOL_VERSION, actorDefinitionVersion.getProtocolVersion()) + .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL, actorDefinitionVersion.getSupportLevel() == null ? null + : Enums.toEnum(actorDefinitionVersion.getSupportLevel().value(), + SupportLevel.class).orElseThrow()) + .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_STAGE, actorDefinitionVersion.getReleaseStage() == null ? null + : Enums.toEnum(actorDefinitionVersion.getReleaseStage().value(), + ReleaseStage.class).orElseThrow()) + .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_DATE, actorDefinitionVersion.getReleaseDate() == null ? null + : LocalDate.parse(actorDefinitionVersion.getReleaseDate())) + .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_REPOSITORY, + Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) + ? actorDefinitionVersion.getNormalizationConfig().getNormalizationRepository() + : null) + .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_TAG, + Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) + ? actorDefinitionVersion.getNormalizationConfig().getNormalizationTag() + : null) + .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORTS_DBT, actorDefinitionVersion.getSupportsDbt()) + .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_INTEGRATION_TYPE, + Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) + ? actorDefinitionVersion.getNormalizationConfig().getNormalizationIntegrationType() + : null) + .set(Tables.ACTOR_DEFINITION_VERSION.ALLOWED_HOSTS, actorDefinitionVersion.getAllowedHosts() == null ? null + : JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getAllowedHosts()))) + .set(Tables.ACTOR_DEFINITION_VERSION.SUGGESTED_STREAMS, + actorDefinitionVersion.getSuggestedStreams() == null ? null + : JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSuggestedStreams()))) + .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_STATE, + Enums.toEnum(actorDefinitionVersion.getSupportState().value(), io.airbyte.db.instance.configs.jooq.generated.enums.SupportState.class) + .orElseThrow()) + .execute(); + + return actorDefinitionVersion.withVersionId(versionId); + } + + /** + * Get the actor definition version associated with an actor definition and a docker image tag. + * + * @param actorDefinitionId - actor definition id + * @param dockerImageTag - docker image tag + * @param ctx database context + * @return actor definition version if there is an entry in the DB already for this version, + * otherwise an empty optional + * @throws IOException - you never know when you io + */ + public static Optional getActorDefinitionVersion(final UUID actorDefinitionId, + final String dockerImageTag, + final DSLContext ctx) { + return ctx.selectFrom(Tables.ACTOR_DEFINITION_VERSION) + .where(Tables.ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID.eq(actorDefinitionId) + .and(Tables.ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG.eq(dockerImageTag))) + .fetch() + .stream() + .findFirst() + .map(DbConverter::buildActorDefinitionVersion); + } + + /** + * Set the ActorDefinitionVersion for a given tag as the default version for the associated actor + * definition. Check docker image tag on the new ADV; if an ADV exists for that tag, set the + * existing ADV for the tag as the default. Otherwise, insert the new ADV and set it as the default. + * + * @param actorDefinitionVersion new actor definition version + * @throws IOException - you never know when you IO + */ + public static void setActorDefinitionVersionForTagAsDefault(final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition, + final DSLContext ctx) { + final Optional existingADV = + getActorDefinitionVersion(actorDefinitionVersion.getActorDefinitionId(), actorDefinitionVersion.getDockerImageTag(), ctx); + + if (existingADV.isPresent()) { + setActorDefinitionVersionAsDefaultVersion(existingADV.get(), breakingChangesForDefinition, ctx); + } else { + final ActorDefinitionVersion insertedADV = writeActorDefinitionVersion(actorDefinitionVersion, ctx); + setActorDefinitionVersionAsDefaultVersion(insertedADV, breakingChangesForDefinition, ctx); + } + } + + private static void setActorDefinitionVersionAsDefaultVersion(final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition, + final DSLContext ctx) { + if (actorDefinitionVersion.getVersionId() == null) { + throw new RuntimeException("Can't set an actorDefinitionVersion as default without it having a versionId."); + } + + final Optional currentDefaultVersion = + getDefaultVersionForActorDefinitionIdOptional(actorDefinitionVersion.getActorDefinitionId(), ctx); + + currentDefaultVersion + .ifPresent(currentDefault -> { + final boolean shouldUpdateActorDefaultVersions = shouldUpdateActorsDefaultVersionsDuringUpgrade( + currentDefault.getDockerImageTag(), actorDefinitionVersion.getDockerImageTag(), breakingChangesForDefinition); + if (shouldUpdateActorDefaultVersions) { + updateDefaultVersionIdForActorsOnVersion(currentDefault.getVersionId(), actorDefinitionVersion.getVersionId(), ctx); + } + }); + + updateActorDefinitionDefaultVersionId(actorDefinitionVersion.getActorDefinitionId(), actorDefinitionVersion.getVersionId(), ctx); + } + + /** + * Get an optional ADV for an actor definition's default version. The optional will be empty if the + * defaultVersionId of the actor definition is set to null in the DB. The only time this should be + * the case is if we are in the process of inserting and have already written the source definition, + * but not yet set its default version. + */ + public static Optional getDefaultVersionForActorDefinitionIdOptional(final UUID actorDefinitionId, final DSLContext ctx) { + return ctx.select(Tables.ACTOR_DEFINITION_VERSION.asterisk()) + .from(ACTOR_DEFINITION) + .join(ACTOR_DEFINITION_VERSION).on(Tables.ACTOR_DEFINITION_VERSION.ID.eq(Tables.ACTOR_DEFINITION.DEFAULT_VERSION_ID)) + .where(ACTOR_DEFINITION.ID.eq(actorDefinitionId)) + .fetch() + .stream() + .findFirst() + .map(DbConverter::buildActorDefinitionVersion); + } + + private static void updateDefaultVersionIdForActorsOnVersion(final UUID previousDefaultVersionId, + final UUID newDefaultVersionId, + final DSLContext ctx) { + ctx.update(ACTOR) + .set(ACTOR.UPDATED_AT, OffsetDateTime.now()) + .set(ACTOR.DEFAULT_VERSION_ID, newDefaultVersionId) + .where(ACTOR.DEFAULT_VERSION_ID.eq(previousDefaultVersionId)) + .execute(); + } + + /** + * Given a current version and a version to upgrade to, and a list of breaking changes, determine + * whether actors' default versions should be updated during upgrade. This logic is used to avoid + * applying a breaking change to a user's actor. + * + * @param currentDockerImageTag version to upgrade from + * @param dockerImageTagForUpgrade version to upgrade to + * @param breakingChangesForDef a list of breaking changes to check + * @return whether actors' default versions should be updated during upgrade + */ + private static boolean shouldUpdateActorsDefaultVersionsDuringUpgrade(final String currentDockerImageTag, + final String dockerImageTagForUpgrade, + final List breakingChangesForDef) { + if (breakingChangesForDef.isEmpty()) { + // If there aren't breaking changes, early exit in order to avoid trying to parse versions. + // This is helpful for custom connectors or local dev images for connectors that don't have + // breaking changes. + return true; + } + + final Version currentVersion = new Version(currentDockerImageTag); + final Version versionToUpgradeTo = new Version(dockerImageTagForUpgrade); + + if (versionToUpgradeTo.lessThanOrEqualTo(currentVersion)) { + // When downgrading, we don't take into account breaking changes/hold actors back. + return true; + } + + final boolean upgradingOverABreakingChange = breakingChangesForDef.stream().anyMatch( + breakingChange -> currentVersion.lessThan(breakingChange.getVersion()) && versionToUpgradeTo.greaterThanOrEqualTo( + breakingChange.getVersion())); + return !upgradingOverABreakingChange; + } + + private static void updateActorDefinitionDefaultVersionId(final UUID actorDefinitionId, final UUID versionId, final DSLContext ctx) { + ctx.update(ACTOR_DEFINITION) + .set(ACTOR_DEFINITION.UPDATED_AT, OffsetDateTime.now()) + .set(ACTOR_DEFINITION.DEFAULT_VERSION_ID, versionId) + .where(ACTOR_DEFINITION.ID.eq(actorDefinitionId)) + .execute(); + } + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/DestinationServiceJooqImpl.java b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/DestinationServiceJooqImpl.java index a6a3e140335..03fa0d6dec9 100644 --- a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/DestinationServiceJooqImpl.java +++ b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/DestinationServiceJooqImpl.java @@ -19,9 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.ConfigSchema; @@ -44,8 +42,6 @@ import io.airbyte.db.ExceptionWrappingDatabase; import io.airbyte.db.instance.configs.jooq.generated.Tables; import io.airbyte.db.instance.configs.jooq.generated.enums.ActorType; -import io.airbyte.db.instance.configs.jooq.generated.enums.ReleaseStage; -import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; import io.airbyte.db.instance.configs.jooq.generated.tables.records.ActorDefinitionWorkspaceGrantRecord; import io.airbyte.db.instance.configs.jooq.generated.tables.records.NotificationConfigurationRecord; import io.airbyte.featureflag.FeatureFlagClient; @@ -63,7 +59,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.function.Function; @@ -478,7 +473,7 @@ private void writeConnectorMetadata(final StandardDestinationDefinition destinat final DSLContext ctx) { writeStandardDestinationDefinition(Collections.singletonList(destinationDefinition), ctx); writeActorDefinitionBreakingChanges(breakingChangesForDefinition, ctx); - setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); + ActorDefinitionVersionJooqHelper.setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); } /** @@ -497,130 +492,6 @@ private void writeActorDefinitionBreakingChanges(final List getActorDefinitionVersion(final UUID actorDefinitionId, - final String dockerImageTag, - final DSLContext ctx) { - return ctx.selectFrom(Tables.ACTOR_DEFINITION_VERSION) - .where(Tables.ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID.eq(actorDefinitionId) - .and(Tables.ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG.eq(dockerImageTag))) - .fetch() - .stream() - .findFirst() - .map(DbConverter::buildActorDefinitionVersion); - } - - /** - * Set the ActorDefinitionVersion for a given tag as the default version for the associated actor - * definition. Check docker image tag on the new ADV; if an ADV exists for that tag, set the - * existing ADV for the tag as the default. Otherwise, insert the new ADV and set it as the default. - * - * @param actorDefinitionVersion new actor definition version - * @throws IOException - you never know when you IO - */ - private void setActorDefinitionVersionForTagAsDefault(final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition, - final DSLContext ctx) { - final Optional existingADV = - getActorDefinitionVersion(actorDefinitionVersion.getActorDefinitionId(), actorDefinitionVersion.getDockerImageTag(), ctx); - - if (existingADV.isPresent()) { - setActorDefinitionVersionAsDefaultVersion(existingADV.get(), breakingChangesForDefinition, ctx); - } else { - final ActorDefinitionVersion insertedADV = writeActorDefinitionVersion(actorDefinitionVersion, ctx); - setActorDefinitionVersionAsDefaultVersion(insertedADV, breakingChangesForDefinition, ctx); - } - } - - /** - * Insert an actor definition version. - * - * @param actorDefinitionVersion - actor definition version to insert - * @param ctx database context - * @throws IOException - you never know when you io - * @returns the POJO associated with the actor definition version inserted. Contains the versionId - * field from the DB. - */ - private ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionVersion actorDefinitionVersion, final DSLContext ctx) { - final OffsetDateTime timestamp = OffsetDateTime.now(); - // Generate a new UUID if one is not provided. Passing an ID is useful for mocks. - final UUID versionId = actorDefinitionVersion.getVersionId() != null ? actorDefinitionVersion.getVersionId() : UUID.randomUUID(); - - ctx.insertInto(Tables.ACTOR_DEFINITION_VERSION) - .set(Tables.ACTOR_DEFINITION_VERSION.ID, versionId) - .set(ACTOR_DEFINITION_VERSION.CREATED_AT, timestamp) - .set(ACTOR_DEFINITION_VERSION.UPDATED_AT, timestamp) - .set(Tables.ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, actorDefinitionVersion.getActorDefinitionId()) - .set(Tables.ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, actorDefinitionVersion.getDockerRepository()) - .set(Tables.ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, actorDefinitionVersion.getDockerImageTag()) - .set(Tables.ACTOR_DEFINITION_VERSION.SPEC, JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSpec()))) - .set(Tables.ACTOR_DEFINITION_VERSION.DOCUMENTATION_URL, actorDefinitionVersion.getDocumentationUrl()) - .set(Tables.ACTOR_DEFINITION_VERSION.PROTOCOL_VERSION, actorDefinitionVersion.getProtocolVersion()) - .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL, actorDefinitionVersion.getSupportLevel() == null ? null - : Enums.toEnum(actorDefinitionVersion.getSupportLevel().value(), - SupportLevel.class).orElseThrow()) - .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_STAGE, actorDefinitionVersion.getReleaseStage() == null ? null - : Enums.toEnum(actorDefinitionVersion.getReleaseStage().value(), - ReleaseStage.class).orElseThrow()) - .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_DATE, actorDefinitionVersion.getReleaseDate() == null ? null - : LocalDate.parse(actorDefinitionVersion.getReleaseDate())) - .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_REPOSITORY, - Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) - ? actorDefinitionVersion.getNormalizationConfig().getNormalizationRepository() - : null) - .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_TAG, - Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) - ? actorDefinitionVersion.getNormalizationConfig().getNormalizationTag() - : null) - .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORTS_DBT, actorDefinitionVersion.getSupportsDbt()) - .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_INTEGRATION_TYPE, - Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) - ? actorDefinitionVersion.getNormalizationConfig().getNormalizationIntegrationType() - : null) - .set(Tables.ACTOR_DEFINITION_VERSION.ALLOWED_HOSTS, actorDefinitionVersion.getAllowedHosts() == null ? null - : JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getAllowedHosts()))) - .set(Tables.ACTOR_DEFINITION_VERSION.SUGGESTED_STREAMS, - actorDefinitionVersion.getSuggestedStreams() == null ? null - : JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSuggestedStreams()))) - .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_STATE, - Enums.toEnum(actorDefinitionVersion.getSupportState().value(), io.airbyte.db.instance.configs.jooq.generated.enums.SupportState.class) - .orElseThrow()) - .execute(); - - return actorDefinitionVersion.withVersionId(versionId); - } - - private void setActorDefinitionVersionAsDefaultVersion(final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition, - final DSLContext ctx) { - if (actorDefinitionVersion.getVersionId() == null) { - throw new RuntimeException("Can't set an actorDefinitionVersion as default without it having a versionId."); - } - - final Optional currentDefaultVersion = - getDefaultVersionForActorDefinitionIdOptional(actorDefinitionVersion.getActorDefinitionId(), ctx); - - currentDefaultVersion - .ifPresent(currentDefault -> { - final boolean shouldUpdateActorDefaultVersions = shouldUpdateActorsDefaultVersionsDuringUpgrade( - currentDefault.getDockerImageTag(), actorDefinitionVersion.getDockerImageTag(), breakingChangesForDefinition); - if (shouldUpdateActorDefaultVersions) { - updateDefaultVersionIdForActorsOnVersion(currentDefault.getVersionId(), actorDefinitionVersion.getVersionId(), ctx); - } - }); - - updateActorDefinitionDefaultVersionId(actorDefinitionVersion.getActorDefinitionId(), actorDefinitionVersion.getVersionId(), ctx); - } - private Stream destDefQuery(final Optional destDefId, final boolean includeTombstone) throws IOException { return database.query(ctx -> ctx.select(ACTOR_DEFINITION.asterisk()) .from(ACTOR_DEFINITION) @@ -632,56 +503,6 @@ private Stream destDefQuery(final Optional .map(DbConverter::buildStandardDestinationDefinition); } - private void updateActorDefinitionDefaultVersionId(final UUID actorDefinitionId, final UUID versionId, final DSLContext ctx) { - ctx.update(ACTOR_DEFINITION) - .set(ACTOR_DEFINITION.UPDATED_AT, OffsetDateTime.now()) - .set(ACTOR_DEFINITION.DEFAULT_VERSION_ID, versionId) - .where(ACTOR_DEFINITION.ID.eq(actorDefinitionId)) - .execute(); - } - - private void updateDefaultVersionIdForActorsOnVersion(final UUID previousDefaultVersionId, final UUID newDefaultVersionId, final DSLContext ctx) { - ctx.update(ACTOR) - .set(ACTOR.UPDATED_AT, OffsetDateTime.now()) - .set(ACTOR.DEFAULT_VERSION_ID, newDefaultVersionId) - .where(ACTOR.DEFAULT_VERSION_ID.eq(previousDefaultVersionId)) - .execute(); - } - - /** - * Given a current version and a version to upgrade to, and a list of breaking changes, determine - * whether actors' default versions should be updated during upgrade. This logic is used to avoid - * applying a breaking change to a user's actor. - * - * @param currentDockerImageTag version to upgrade from - * @param dockerImageTagForUpgrade version to upgrade to - * @param breakingChangesForDef a list of breaking changes to check - * @return whether actors' default versions should be updated during upgrade - */ - private static boolean shouldUpdateActorsDefaultVersionsDuringUpgrade(final String currentDockerImageTag, - final String dockerImageTagForUpgrade, - final List breakingChangesForDef) { - if (breakingChangesForDef.isEmpty()) { - // If there aren't breaking changes, early exit in order to avoid trying to parse versions. - // This is helpful for custom connectors or local dev images for connectors that don't have - // breaking changes. - return true; - } - - final Version currentVersion = new Version(currentDockerImageTag); - final Version versionToUpgradeTo = new Version(dockerImageTagForUpgrade); - - if (versionToUpgradeTo.lessThanOrEqualTo(currentVersion)) { - // When downgrading, we don't take into account breaking changes/hold actors back. - return true; - } - - final boolean upgradingOverABreakingChange = breakingChangesForDef.stream().anyMatch( - breakingChange -> currentVersion.lessThan(breakingChange.getVersion()) && versionToUpgradeTo.greaterThanOrEqualTo( - breakingChange.getVersion())); - return !upgradingOverABreakingChange; - } - private Query upsertBreakingChangeQuery(final DSLContext ctx, final ActorDefinitionBreakingChange breakingChange, final OffsetDateTime timestamp) { return ctx.insertInto(Tables.ACTOR_DEFINITION_BREAKING_CHANGE) .set(Tables.ACTOR_DEFINITION_BREAKING_CHANGE.ACTOR_DEFINITION_ID, breakingChange.getActorDefinitionId()) diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/SourceServiceJooqImpl.java b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/SourceServiceJooqImpl.java index a173d0390fb..8105c81b220 100644 --- a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/SourceServiceJooqImpl.java +++ b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/SourceServiceJooqImpl.java @@ -6,7 +6,6 @@ import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION; -import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION_VERSION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION_WORKSPACE_GRANT; import static io.airbyte.db.instance.configs.jooq.generated.Tables.CONNECTION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.CONNECTION_OPERATION; @@ -21,7 +20,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.ConfigSchema; @@ -44,9 +42,7 @@ import io.airbyte.db.ExceptionWrappingDatabase; import io.airbyte.db.instance.configs.jooq.generated.Tables; import io.airbyte.db.instance.configs.jooq.generated.enums.ActorType; -import io.airbyte.db.instance.configs.jooq.generated.enums.ReleaseStage; import io.airbyte.db.instance.configs.jooq.generated.enums.SourceType; -import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; import io.airbyte.db.instance.configs.jooq.generated.tables.records.ActorDefinitionWorkspaceGrantRecord; import io.airbyte.db.instance.configs.jooq.generated.tables.records.NotificationConfigurationRecord; import io.airbyte.featureflag.FeatureFlagClient; @@ -64,7 +60,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.function.Function; @@ -479,7 +474,7 @@ private void writeConnectorMetadata(final StandardSourceDefinition sourceDefinit final DSLContext ctx) { writeStandardSourceDefinition(Collections.singletonList(sourceDefinition), ctx); writeActorDefinitionBreakingChanges(breakingChangesForDefinition, ctx); - setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); + ActorDefinitionVersionJooqHelper.setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); } /** @@ -514,169 +509,6 @@ private Query upsertBreakingChangeQuery(final DSLContext ctx, final ActorDefinit .set(Tables.ACTOR_DEFINITION_BREAKING_CHANGE.UPDATED_AT, timestamp); } - /** - * Set the ActorDefinitionVersion for a given tag as the default version for the associated actor - * definition. Check docker image tag on the new ADV; if an ADV exists for that tag, set the - * existing ADV for the tag as the default. Otherwise, insert the new ADV and set it as the default. - * - * @param actorDefinitionVersion new actor definition version - * @throws IOException - you never know when you IO - */ - private void setActorDefinitionVersionForTagAsDefault(final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition, - final DSLContext ctx) { - final Optional existingADV = - getActorDefinitionVersion(actorDefinitionVersion.getActorDefinitionId(), actorDefinitionVersion.getDockerImageTag(), ctx); - - if (existingADV.isPresent()) { - setActorDefinitionVersionAsDefaultVersion(existingADV.get(), breakingChangesForDefinition, ctx); - } else { - final ActorDefinitionVersion insertedADV = writeActorDefinitionVersion(actorDefinitionVersion, ctx); - setActorDefinitionVersionAsDefaultVersion(insertedADV, breakingChangesForDefinition, ctx); - } - } - - private ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionVersion actorDefinitionVersion, final DSLContext ctx) { - final OffsetDateTime timestamp = OffsetDateTime.now(); - // Generate a new UUID if one is not provided. Passing an ID is useful for mocks. - final UUID versionId = actorDefinitionVersion.getVersionId() != null ? actorDefinitionVersion.getVersionId() : UUID.randomUUID(); - - ctx.insertInto(Tables.ACTOR_DEFINITION_VERSION) - .set(Tables.ACTOR_DEFINITION_VERSION.ID, versionId) - .set(ACTOR_DEFINITION_VERSION.CREATED_AT, timestamp) - .set(ACTOR_DEFINITION_VERSION.UPDATED_AT, timestamp) - .set(Tables.ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, actorDefinitionVersion.getActorDefinitionId()) - .set(Tables.ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, actorDefinitionVersion.getDockerRepository()) - .set(Tables.ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, actorDefinitionVersion.getDockerImageTag()) - .set(Tables.ACTOR_DEFINITION_VERSION.SPEC, JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSpec()))) - .set(Tables.ACTOR_DEFINITION_VERSION.DOCUMENTATION_URL, actorDefinitionVersion.getDocumentationUrl()) - .set(Tables.ACTOR_DEFINITION_VERSION.PROTOCOL_VERSION, actorDefinitionVersion.getProtocolVersion()) - .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL, actorDefinitionVersion.getSupportLevel() == null ? null - : Enums.toEnum(actorDefinitionVersion.getSupportLevel().value(), - SupportLevel.class).orElseThrow()) - .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_STAGE, actorDefinitionVersion.getReleaseStage() == null ? null - : Enums.toEnum(actorDefinitionVersion.getReleaseStage().value(), - ReleaseStage.class).orElseThrow()) - .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_DATE, actorDefinitionVersion.getReleaseDate() == null ? null - : LocalDate.parse(actorDefinitionVersion.getReleaseDate())) - .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_REPOSITORY, - Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) - ? actorDefinitionVersion.getNormalizationConfig().getNormalizationRepository() - : null) - .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_TAG, - Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) - ? actorDefinitionVersion.getNormalizationConfig().getNormalizationTag() - : null) - .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORTS_DBT, actorDefinitionVersion.getSupportsDbt()) - .set(Tables.ACTOR_DEFINITION_VERSION.NORMALIZATION_INTEGRATION_TYPE, - Objects.nonNull(actorDefinitionVersion.getNormalizationConfig()) - ? actorDefinitionVersion.getNormalizationConfig().getNormalizationIntegrationType() - : null) - .set(Tables.ACTOR_DEFINITION_VERSION.ALLOWED_HOSTS, actorDefinitionVersion.getAllowedHosts() == null ? null - : JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getAllowedHosts()))) - .set(Tables.ACTOR_DEFINITION_VERSION.SUGGESTED_STREAMS, - actorDefinitionVersion.getSuggestedStreams() == null ? null - : JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSuggestedStreams()))) - .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_STATE, - Enums.toEnum(actorDefinitionVersion.getSupportState().value(), io.airbyte.db.instance.configs.jooq.generated.enums.SupportState.class) - .orElseThrow()) - .execute(); - - return actorDefinitionVersion.withVersionId(versionId); - } - - private void setActorDefinitionVersionAsDefaultVersion(final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition, - final DSLContext ctx) { - if (actorDefinitionVersion.getVersionId() == null) { - throw new RuntimeException("Can't set an actorDefinitionVersion as default without it having a versionId."); - } - - final Optional currentDefaultVersion = - getDefaultVersionForActorDefinitionIdOptional(actorDefinitionVersion.getActorDefinitionId(), ctx); - - currentDefaultVersion - .ifPresent(currentDefault -> { - final boolean shouldUpdateActorDefaultVersions = shouldUpdateActorsDefaultVersionsDuringUpgrade( - currentDefault.getDockerImageTag(), actorDefinitionVersion.getDockerImageTag(), breakingChangesForDefinition); - if (shouldUpdateActorDefaultVersions) { - updateDefaultVersionIdForActorsOnVersion(currentDefault.getVersionId(), actorDefinitionVersion.getVersionId(), ctx); - } - }); - - updateActorDefinitionDefaultVersionId(actorDefinitionVersion.getActorDefinitionId(), actorDefinitionVersion.getVersionId(), ctx); - } - - private void updateActorDefinitionDefaultVersionId(final UUID actorDefinitionId, final UUID versionId, final DSLContext ctx) { - ctx.update(ACTOR_DEFINITION) - .set(ACTOR_DEFINITION.UPDATED_AT, OffsetDateTime.now()) - .set(ACTOR_DEFINITION.DEFAULT_VERSION_ID, versionId) - .where(ACTOR_DEFINITION.ID.eq(actorDefinitionId)) - .execute(); - } - - private void updateDefaultVersionIdForActorsOnVersion(final UUID previousDefaultVersionId, final UUID newDefaultVersionId, final DSLContext ctx) { - ctx.update(ACTOR) - .set(ACTOR.UPDATED_AT, OffsetDateTime.now()) - .set(ACTOR.DEFAULT_VERSION_ID, newDefaultVersionId) - .where(ACTOR.DEFAULT_VERSION_ID.eq(previousDefaultVersionId)) - .execute(); - } - - /** - * Given a current version and a version to upgrade to, and a list of breaking changes, determine - * whether actors' default versions should be updated during upgrade. This logic is used to avoid - * applying a breaking change to a user's actor. - * - * @param currentDockerImageTag version to upgrade from - * @param dockerImageTagForUpgrade version to upgrade to - * @param breakingChangesForDef a list of breaking changes to check - * @return whether actors' default versions should be updated during upgrade - */ - public static boolean shouldUpdateActorsDefaultVersionsDuringUpgrade(final String currentDockerImageTag, - final String dockerImageTagForUpgrade, - final List breakingChangesForDef) { - if (breakingChangesForDef.isEmpty()) { - // If there aren't breaking changes, early exit in order to avoid trying to parse versions. - // This is helpful for custom connectors or local dev images for connectors that don't have - // breaking changes. - return true; - } - - final Version currentVersion = new Version(currentDockerImageTag); - final Version versionToUpgradeTo = new Version(dockerImageTagForUpgrade); - - if (versionToUpgradeTo.lessThanOrEqualTo(currentVersion)) { - // When downgrading, we don't take into account breaking changes/hold actors back. - return true; - } - - final boolean upgradingOverABreakingChange = breakingChangesForDef.stream().anyMatch( - breakingChange -> currentVersion.lessThan(breakingChange.getVersion()) && versionToUpgradeTo.greaterThanOrEqualTo( - breakingChange.getVersion())); - return !upgradingOverABreakingChange; - } - - /** - * Get the actor definition version associated with an actor definition and a docker image tag. - * - * @param actorDefinitionId - actor definition id - * @param dockerImageTag - docker image tag - * @param ctx database context - * @return actor definition version if there is an entry in the DB already for this version, - * otherwise an empty optional - * @throws IOException - you never know when you io - */ - public Optional getActorDefinitionVersion(final UUID actorDefinitionId, final String dockerImageTag, final DSLContext ctx) { - return ctx.selectFrom(Tables.ACTOR_DEFINITION_VERSION) - .where(Tables.ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID.eq(actorDefinitionId) - .and(Tables.ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG.eq(dockerImageTag))) - .fetch() - .stream() - .findFirst() - .map(DbConverter::buildActorDefinitionVersion); - } - private ConfigWithMetadata getStandardSyncWithMetadata(final UUID connectionId) throws IOException, ConfigNotFoundException { final List> result = listStandardSyncWithMetadata(Optional.of(connectionId)); @@ -934,14 +766,7 @@ private ActorDefinitionVersion getDefaultVersionForActorDefinitionId(final UUID * but not yet set its default version. */ private Optional getDefaultVersionForActorDefinitionIdOptional(final UUID actorDefinitionId, final DSLContext ctx) { - return ctx.select(Tables.ACTOR_DEFINITION_VERSION.asterisk()) - .from(ACTOR_DEFINITION) - .join(ACTOR_DEFINITION_VERSION).on(Tables.ACTOR_DEFINITION_VERSION.ID.eq(Tables.ACTOR_DEFINITION.DEFAULT_VERSION_ID)) - .where(ACTOR_DEFINITION.ID.eq(actorDefinitionId)) - .fetch() - .stream() - .findFirst() - .map(DbConverter::buildActorDefinitionVersion); + return ActorDefinitionVersionJooqHelper.getDefaultVersionForActorDefinitionIdOptional(actorDefinitionId, ctx); } /** diff --git a/airbyte-db/db-lib/build.gradle b/airbyte-db/db-lib/build.gradle deleted file mode 100644 index a0fcf9b8f20..00000000000 --- a/airbyte-db/db-lib/build.gradle +++ /dev/null @@ -1,115 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -// Add a configuration for our migrations tasks defined below to encapsulate their dependencies -configurations { - migrations.extendsFrom implementation -} - -configurations.all { - exclude group: 'io.micronaut.flyway' - resolutionStrategy { - force libs.platform.testcontainers.postgresql - } -} - -airbyte { - docker { - imageName = "db" - } -} - -dependencies { - api libs.hikaricp - api libs.jooq.meta - api libs.jooq - api libs.postgresql - - implementation project(':airbyte-commons') - implementation libs.airbyte.protocol - implementation project(':airbyte-json-validation') - implementation project(':airbyte-config:config-models') - implementation libs.flyway.core - implementation libs.guava - implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) - implementation libs.bundles.jackson - implementation libs.commons.io - - migrations libs.platform.testcontainers.postgresql - migrations sourceSets.main.output - - // Mark as compile only to avoid leaking transitively to connectors - compileOnly libs.platform.testcontainers.postgresql - - // These are required because gradle might be using lower version of Jna from other - // library transitive dependency. Can be removed if we can figure out which library is the cause. - // Refer: https://github.com/testcontainers/testcontainers-java/issues/3834#issuecomment-825409079 - implementation 'net.java.dev.jna:jna:5.8.0' - implementation 'net.java.dev.jna:jna-platform:5.8.0' - - testImplementation project(':airbyte-test-utils') - testImplementation libs.apache.commons.lang - testImplementation libs.platform.testcontainers.postgresql - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer - testImplementation libs.json.assert -} - -tasks.register("newConfigsMigration", JavaExec) { - mainClass = 'io.airbyte.db.instance.development.MigrationDevCenter' - classpath = files(configurations.migrations.files) - args 'configs', 'create' - dependsOn(tasks.named("classes")) -} - -tasks.register("runConfigsMigration", JavaExec) { - mainClass = 'io.airbyte.db.instance.development.MigrationDevCenter' - classpath = files(configurations.migrations.files) - args 'configs', 'migrate' - dependsOn(tasks.named("classes")) -} - -tasks.register("dumpConfigsSchema", JavaExec) { - mainClass = 'io.airbyte.db.instance.development.MigrationDevCenter' - classpath = files(configurations.migrations.files) - args 'configs', 'dump_schema' - dependsOn(tasks.named("classes")) -} - -tasks.register("newJobsMigration", JavaExec) { - mainClass = 'io.airbyte.db.instance.development.MigrationDevCenter' - classpath = files(configurations.migrations.files) - args 'jobs', 'create' - dependsOn(tasks.named("classes")) -} - -tasks.register("runJobsMigration", JavaExec) { - mainClass = 'io.airbyte.db.instance.development.MigrationDevCenter' - classpath = files(configurations.migrations.files) - args 'jobs', 'migrate' - dependsOn(tasks.named("classes")) -} - -tasks.register("dumpJobsSchema", JavaExec) { - mainClass = 'io.airbyte.db.instance.development.MigrationDevCenter' - classpath = files(configurations.migrations.files) - args 'jobs', 'dump_schema' - dependsOn(tasks.named("classes")) -} - -def copyInitSql = tasks.register("copyInitSql", Copy) { - from('src/main/resources') { - include 'init.sql' - } - into 'build/airbyte/docker/bin' -} - -tasks.named("dockerBuildImage") { - dependsOn copyInitSql -} diff --git a/airbyte-db/db-lib/build.gradle.kts b/airbyte-db/db-lib/build.gradle.kts new file mode 100644 index 00000000000..6e412f404df --- /dev/null +++ b/airbyte-db/db-lib/build.gradle.kts @@ -0,0 +1,115 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +// Add a configuration(for our migrations(tasks defined below to encapsulate their dependencies) +val migrations by configurations.creating { + extendsFrom(configurations.getByName("implementation")) +} + +configurations.all { + exclude(group = "io.micronaut.flyway") + resolutionStrategy { + force (libs.platform.testcontainers.postgresql) + } +} + +airbyte { + docker { + imageName = "db" + } +} + +dependencies { + api(libs.hikaricp) + api(libs.jooq.meta) + api(libs.jooq) + api(libs.postgresql) + + implementation(project(":airbyte-commons")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-json-validation")) + implementation(project(":airbyte-config:config-models")) + implementation(libs.flyway.core) + implementation(libs.guava) + implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) + implementation(libs.bundles.jackson) + implementation(libs.commons.io) + + migrations(libs.platform.testcontainers.postgresql) + migrations(sourceSets["main"].output) + + // Mark as compile Only to avoid leaking transitively to connectors) + compileOnly(libs.platform.testcontainers.postgresql) + + // These are required because gradle might be using lower version of Jna from other) + // library transitive dependency. Can be removed if we can figure out which library is the cause.) + // Refer: https://github.com/testcontainers/testcontainers-java/issues/3834#issuecomment-825409079) + implementation("net.java.dev.jna:jna:5.8.0") + implementation("net.java.dev.jna:jna-platform:5.8.0") + + testImplementation(project(":airbyte-test-utils")) + testImplementation(libs.apache.commons.lang) + testImplementation(libs.platform.testcontainers.postgresql) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) + testImplementation(libs.json.assert) +} + +tasks.register("newConfigsMigration") { + mainClass = "io.airbyte.db.instance.development.MigrationDevCenter" + classpath = files(migrations.files) + args = listOf("configs", "create") + dependsOn(tasks.named("classes")) +} + +tasks.register("runConfigsMigration") { + mainClass = "io.airbyte.db.instance.development.MigrationDevCenter" + classpath = files(migrations.files) + args = listOf("configs", "migrate") + dependsOn(tasks.named("classes")) +} + +tasks.register("dumpConfigsSchema") { + mainClass = "io.airbyte.db.instance.development.MigrationDevCenter" + classpath = files(migrations.files) + args = listOf("configs", "dump_schema") + dependsOn(tasks.named("classes")) +} + +tasks.register("newJobsMigration") { + mainClass = "io.airbyte.db.instance.development.MigrationDevCenter" + classpath = files(migrations.files) + args = listOf("jobs", "create") + dependsOn(tasks.named("classes")) +} + +tasks.register("runJobsMigration") { + mainClass = "io.airbyte.db.instance.development.MigrationDevCenter" + classpath = files(migrations.files) + args = listOf( "jobs", "migrate") + dependsOn(tasks.named("classes")) +} + +tasks.register("dumpJobsSchema") { + mainClass = "io.airbyte.db.instance.development.MigrationDevCenter" + classpath = files(migrations.files) + args = listOf("jobs", "dump_schema") + dependsOn(tasks.named("classes")) +} + +val copyInitSql = tasks.register("copyInitSql") { + from("src/main/resources") { + include("init.sql") + } + into("build/airbyte/docker/bin") +} + +tasks.named("dockerBuildImage") { + dependsOn(copyInitSql) +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_007__AddGeographyColumnToWorkload.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_007__AddGeographyColumnToWorkload.java new file mode 100644 index 00000000000..b3bf6835c74 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_007__AddGeographyColumnToWorkload.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class V0_50_33_007__AddGeographyColumnToWorkload extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_33_007__AddGeographyColumnToWorkload.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + + addGeographyColumnToWorkload(ctx); + } + + private static void addGeographyColumnToWorkload(final DSLContext ctx) { + ctx.alterTable("workload") + .addColumnIfNotExists(DSL.field( + "geography", + SQLDataType.VARCHAR.nullable(false).defaultValue("AUTO"))) + .execute(); + } + +} diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt index 48620c52ac4..8db8a9ee847 100644 --- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt @@ -359,6 +359,7 @@ create table "public"."workload" ( "last_heartbeat_at" timestamp(6) with time zone, "input_payload" text not null, "log_path" text not null, + "geography" varchar(2147483647) not null default cast('AUTO' as varchar), constraint "workload_pkey" primary key ("id") ); diff --git a/airbyte-db/jooq/build.gradle b/airbyte-db/jooq/build.gradle deleted file mode 100644 index 4374047f9d9..00000000000 --- a/airbyte-db/jooq/build.gradle +++ /dev/null @@ -1,98 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - alias(libs.plugins.nu.studer.jooq) -} - -configurations.all { - resolutionStrategy { - force libs.platform.testcontainers.postgresql - } -} -dependencies { - implementation libs.jooq.meta - implementation libs.jooq - implementation libs.postgresql - implementation libs.flyway.core - implementation project(':airbyte-db:db-lib') - - // jOOQ code generation - implementation libs.jooq.codegen - implementation libs.platform.testcontainers.postgresql - - // These are required because gradle might be using lower version of Jna from other - // library transitive dependency. Can be removed if we can figure out which library is the cause. - // Refer: https://github.com/testcontainers/testcontainers-java/issues/3834#issuecomment-825409079 - implementation 'net.java.dev.jna:jna:5.8.0' - implementation 'net.java.dev.jna:jna-platform:5.8.0' - - // The jOOQ code generator only has access to classes added to the jooqGenerator configuration - jooqGenerator project(':airbyte-db:db-lib') - jooqGenerator libs.platform.testcontainers.postgresql -} - -jooq { - version = libs.versions.jooq - edition = nu.studer.gradle.jooq.JooqEdition.OSS - - configurations { - configsDatabase { - generateSchemaSourceOnCompilation = true - generationTool { - generator { - name = 'org.jooq.codegen.DefaultGenerator' - database { - name = 'io.airbyte.db.instance.configs.ConfigsFlywayMigrationDatabase' - inputSchema = 'public' - excludes = 'airbyte_configs_migrations' - } - target { - packageName = 'io.airbyte.db.instance.configs.jooq.generated' - directory = 'build/generated/configsDatabase/src/main/java' - } - } - } - } - - jobsDatabase { - generateSchemaSourceOnCompilation = true - generationTool { - generator { - name = 'org.jooq.codegen.DefaultGenerator' - database { - name = 'io.airbyte.db.instance.jobs.JobsFlywayMigrationDatabase' - inputSchema = 'public' - excludes = 'airbyte_jobs_migrations' - } - target { - packageName = 'io.airbyte.db.instance.jobs.jooq.generated' - directory = 'build/generated/jobsDatabase/src/main/java' - } - } - } - } - } -} - -sourceSets.main.java.srcDirs( - tasks.named('generateConfigsDatabaseJooq').flatMap { it.outputDir }, - tasks.named('generateJobsDatabaseJooq').flatMap { it.outputDir } -) - -sourceSets { - main { - java { - srcDirs "$buildDir/generated/configsDatabase/src/main/java", "$buildDir/generated/jobsDatabase/src/main/java" - } - } -} - -tasks.named('generateConfigsDatabaseJooq') { - allInputsDeclared = true - outputs.cacheIf { true } -} - -tasks.named('generateJobsDatabaseJooq') { - allInputsDeclared = true - outputs.cacheIf { true } -} diff --git a/airbyte-db/jooq/build.gradle.kts b/airbyte-db/jooq/build.gradle.kts new file mode 100644 index 00000000000..103ff3d48d8 --- /dev/null +++ b/airbyte-db/jooq/build.gradle.kts @@ -0,0 +1,99 @@ +import nu.studer.gradle.jooq.JooqGenerate + +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + alias(libs.plugins.nu.studer.jooq) +} + +configurations.all { + resolutionStrategy { + force(libs.platform.testcontainers.postgresql) + } +} +dependencies { + implementation(libs.jooq.meta) + implementation(libs.jooq) + implementation(libs.postgresql) + implementation(libs.flyway.core) + implementation(project(":airbyte-db:db-lib")) + + // jOOQ code generation) + implementation(libs.jooq.codegen) + implementation(libs.platform.testcontainers.postgresql) + + // These are required because gradle might be using lower version of Jna from other + // library transitive dependency. Can be removed if we can figure out which library is the cause. + // Refer: https://github.com/testcontainers/testcontainers-java/issues/3834#issuecomment-825409079 + implementation("net.java.dev.jna:jna:5.8.0") + implementation("net.java.dev.jna:jna-platform:5.8.0") + + // The jOOQ code generator(only has access to classes added to the jooqGenerator configuration + jooqGenerator(project(":airbyte-db:db-lib")) + jooqGenerator(libs.platform.testcontainers.postgresql) +} + +jooq { + version = libs.versions.jooq + edition = nu.studer.gradle.jooq.JooqEdition.OSS + + configurations { + create("configsDatabase") { + generateSchemaSourceOnCompilation = true + jooqConfiguration.apply { + generator.apply { + name = "org.jooq.codegen.DefaultGenerator" + database.apply { + name = "io.airbyte.db.instance.configs.ConfigsFlywayMigrationDatabase" + inputSchema = "public" + excludes = "airbyte_configs_migrations" + } + target.apply { + packageName = "io.airbyte.db.instance.configs.jooq.generated" + directory = "build/generated/configsDatabase/src/main/java" + } + } + } + } + + create("jobsDatabase") { + generateSchemaSourceOnCompilation = true + jooqConfiguration.apply { + generator.apply { + name = "org.jooq.codegen.DefaultGenerator" + database.apply { + name = "io.airbyte.db.instance.jobs.JobsFlywayMigrationDatabase" + inputSchema = "public" + excludes = "airbyte_jobs_migrations" + } + target.apply { + packageName = "io.airbyte.db.instance.jobs.jooq.generated" + directory = "build/generated/jobsDatabase/src/main/java" + } + } + } + } + } +} + +sourceSets["main"].java { + srcDirs( + tasks.named("generateConfigsDatabaseJooq").flatMap { it.outputDir }, + tasks.named("generateJobsDatabaseJooq").flatMap { it.outputDir }, + ) +} + + +sourceSets["main"].java { + srcDirs("$buildDir/generated/configsDatabase/src/main/java", "$buildDir/generated/jobsDatabase/src/main/java") +} + +tasks.named("generateConfigsDatabaseJooq") { + allInputsDeclared = true + outputs.cacheIf { true } +} + +tasks.named("generateJobsDatabaseJooq") { + allInputsDeclared = true + outputs.cacheIf { true } +} diff --git a/airbyte-featureflag/build.gradle.kts b/airbyte-featureflag/build.gradle.kts index d4224120b27..d2492b6d1d1 100644 --- a/airbyte-featureflag/build.gradle.kts +++ b/airbyte-featureflag/build.gradle.kts @@ -1,5 +1,3 @@ -// @Suppress can be removed when KTIJ-19369 has been fixed, or when we upgrade to gradle 8.1 -@Suppress("DSL_SCOPE_VIOLATION") plugins { id("io.airbyte.gradle.jvm.lib") id("io.airbyte.gradle.publish") diff --git a/airbyte-featureflag/src/main/kotlin/Context.kt b/airbyte-featureflag/src/main/kotlin/Context.kt index 01534195693..9e9d3eca318 100644 --- a/airbyte-featureflag/src/main/kotlin/Context.kt +++ b/airbyte-featureflag/src/main/kotlin/Context.kt @@ -209,3 +209,7 @@ data class ImageName(override val key: String) : Context { data class ImageVersion(override val key: String) : Context { override val kind = "image-version" } + +data class Geography(override val key: String) : Context { + override val kind: String = "geography" +} diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index b4cbfa4c737..c08aa044ce5 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -31,8 +31,6 @@ object ContainerOrchestratorJavaOpts : Temporary(key = "container-orches object NewTrialPolicyEnabled : Temporary(key = "billing.newTrialPolicy", default = false) -object AutoPropagateNewStreams : Temporary(key = "autopropagate-new-streams.enabled", default = false) - object CanonicalCatalogSchema : Temporary(key = "canonical-catalog-schema", default = false) object CatalogCanonicalJson : Temporary(key = "catalog-canonical-json", default = false) @@ -61,9 +59,11 @@ object ShouldFailSyncIfHeartbeatFailure : Permanent(key = "heartbeat.fa object ConnectorVersionOverride : Permanent(key = "connectors.versionOverrides", default = "") -object DestinationTimeoutEnabled : Permanent(key = "destination-timeout-enabled", default = false) +object DestinationTimeoutEnabled : Permanent(key = "destination-timeout-enabled", default = true) + +object ShouldFailSyncOnDestinationTimeout : Permanent(key = "destination-timeout.failSync", default = true) -object ShouldFailSyncOnDestinationTimeout : Permanent(key = "destination-timeout.failSync", default = false) +object DestinationTimeoutSeconds : Permanent(key = "destination-timeout.seconds", default = 7200) object UseActorScopedDefaultVersions : Temporary(key = "connectors.useActorScopedDefaultVersions", default = true) @@ -159,3 +159,9 @@ object UseNewCronScheduleCalculation : Temporary(key = "platform.use-ne object UseRuntimeSecretPersistence : Temporary(key = "platform.use-runtime-secret-persistence", default = false) object UseWorkloadApi : Temporary(key = "platform.use-workload-api", default = false) + +object WorkloadApiRouting : Permanent(key = "workload-api-routing", default = "workload_default") + +object FailMissingPks : Temporary(key = "platform.fail-missing-pks", default = false) + +object PrintLongRecordPks : Temporary(key = "platform.print-long-record-pks", default = false) diff --git a/airbyte-json-validation/build.gradle b/airbyte-json-validation/build.gradle deleted file mode 100644 index 167dd29b523..00000000000 --- a/airbyte-json-validation/build.gradle +++ /dev/null @@ -1,18 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - implementation project(':airbyte-commons') - implementation libs.guava - implementation 'com.networknt:json-schema-validator:1.0.72' - // needed so that we can follow $ref when parsing json. jackson does not support this natively. - implementation 'me.andrz.jackson:jackson-json-reference-core:0.3.2' - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer -} diff --git a/airbyte-json-validation/build.gradle.kts b/airbyte-json-validation/build.gradle.kts new file mode 100644 index 00000000000..fed6824bd5a --- /dev/null +++ b/airbyte-json-validation/build.gradle.kts @@ -0,0 +1,18 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + implementation(project(":airbyte-commons")) + implementation(libs.guava) + implementation("com.networknt:json-schema-validator:1.0.72") + // needed so that we can follow $ref when parsing json. jackson does not support this natively. + implementation("me.andrz.jackson:jackson-json-reference-core:0.3.2") + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) +} diff --git a/airbyte-keycloak-setup/build.gradle b/airbyte-keycloak-setup/build.gradle deleted file mode 100644 index 7fd637efd6c..00000000000 --- a/airbyte-keycloak-setup/build.gradle +++ /dev/null @@ -1,49 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.app" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.bundles.keycloak.client - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-auth') - implementation project(':airbyte-commons-micronaut') - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - - testImplementation libs.bundles.micronaut.test - testImplementation libs.bundles.junit - testImplementation libs.junit.jupiter.system.stubs -} - -application { - applicationName = project.name - mainClass = 'io.airbyte.keycloak.setup.Application' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -Properties env = new Properties() -rootProject.file('.env.dev').withInputStream { env.load(it) } - -airbyte { - application { - mainClass = 'io.airbyte.keycloak.setup.Application' - defaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] - } - - docker { - imageName = "keycloak-setup" - } -} diff --git a/airbyte-keycloak-setup/build.gradle.kts b/airbyte-keycloak-setup/build.gradle.kts new file mode 100644 index 00000000000..6334c087801 --- /dev/null +++ b/airbyte-keycloak-setup/build.gradle.kts @@ -0,0 +1,44 @@ +import java.util.Properties + +plugins { + id("io.airbyte.gradle.jvm.app") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation( platform(libs.micronaut.bom)) + implementation( libs.bundles.micronaut) + implementation( libs.bundles.keycloak.client) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-auth")) + implementation(project(":airbyte-commons-micronaut")) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.bundles.junit) + testImplementation(libs.junit.jupiter.system.stubs) +} + +val env = Properties().apply { + load(rootProject.file(".env.dev").inputStream()) +} +airbyte { + application { + mainClass = "io.airbyte.keycloak.setup.Application" + defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + } + docker { + imageName = "keycloak-setup" + } +} diff --git a/airbyte-keycloak/build.gradle b/airbyte-keycloak/build.gradle deleted file mode 100644 index 565e3d57632..00000000000 --- a/airbyte-keycloak/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -plugins { - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -airbyte { - docker { - imageName = "keycloak" - } -} - -def copyTheme = tasks.register("copyTheme", Copy) { - from('themes') - into 'build/airbyte/docker/bin/themes' -} - -def copyScripts = tasks.register("copyScripts", Copy) { - from('scripts') - into 'build/airbyte/docker/bin/scripts' -} - -tasks.named("dockerBuildImage") { - dependsOn copyScripts - dependsOn copyTheme -} diff --git a/airbyte-keycloak/build.gradle.kts b/airbyte-keycloak/build.gradle.kts new file mode 100644 index 00000000000..1c44ffcc4f2 --- /dev/null +++ b/airbyte-keycloak/build.gradle.kts @@ -0,0 +1,24 @@ +plugins { + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +airbyte { + docker { + imageName = "keycloak" + } +} + +val copyTheme = tasks.register("copyTheme") { + from("themes") + into("build/airbyte/docker/bin/themes") +} + +val copyScripts = tasks.register("copyScripts") { + from("scripts") + into("build/airbyte/docker/bin/scripts") +} + +tasks.named("dockerBuildImage") { + dependsOn(copyScripts, copyTheme) +} diff --git a/airbyte-metrics/metrics-lib/build.gradle b/airbyte-metrics/metrics-lib/build.gradle deleted file mode 100644 index 73fa2822766..00000000000 --- a/airbyte-metrics/metrics-lib/build.gradle +++ /dev/null @@ -1,39 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - implementation project(':airbyte-commons') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-db:jooq') - implementation project(':airbyte-db:db-lib') - - implementation libs.guava - implementation libs.google.cloud.storage - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation libs.otel.semconv - implementation libs.otel.sdk - implementation libs.otel.sdk.testing - implementation libs.micrometer.statsd - implementation platform(libs.otel.bom) - implementation("io.opentelemetry:opentelemetry-api") - implementation("io.opentelemetry:opentelemetry-sdk") - implementation("io.opentelemetry:opentelemetry-exporter-otlp") - - implementation libs.java.dogstatsd.client - implementation libs.bundles.datadog - - testImplementation project(':airbyte-config:config-persistence') - testImplementation project(':airbyte-test-utils') - testImplementation libs.platform.testcontainers.postgresql - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - testImplementation(variantOf(libs.opentracing.util.test) { classifier('tests') }) - - testImplementation libs.junit.pioneer - -} diff --git a/airbyte-metrics/metrics-lib/build.gradle.kts b/airbyte-metrics/metrics-lib/build.gradle.kts new file mode 100644 index 00000000000..b740ae5f33b --- /dev/null +++ b/airbyte-metrics/metrics-lib/build.gradle.kts @@ -0,0 +1,39 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + implementation( project(":airbyte-commons")) + implementation( project(":airbyte-config:config-models")) + implementation( project(":airbyte-db:jooq")) + implementation( project(":airbyte-db:db-lib")) + + implementation( libs.guava) + implementation( libs.google.cloud.storage) + compileOnly(libs.lombok) + annotationProcessor( libs.lombok) + + implementation( libs.otel.semconv) + implementation( libs.otel.sdk) + implementation( libs.otel.sdk.testing) + implementation( libs.micrometer.statsd) + implementation( platform(libs.otel.bom)) + implementation(("io.opentelemetry:opentelemetry-api")) + implementation(("io.opentelemetry:opentelemetry-sdk")) + implementation(("io.opentelemetry:opentelemetry-exporter-otlp")) + + implementation( libs.java.dogstatsd.client) + implementation( libs.bundles.datadog) + + testImplementation( project(":airbyte-config:config-persistence")) + testImplementation( project(":airbyte-test-utils")) + testImplementation( libs.platform.testcontainers.postgresql) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation( libs.bundles.junit) + testImplementation( libs.assertj.core) + testImplementation((variantOf(libs.opentracing.util.test) { classifier("tests") })) + + testImplementation( libs.junit.pioneer) + +} diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java index 567c990d00c..abf181d3be2 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java @@ -17,6 +17,9 @@ * - Use dashes to delimit application names with multiple words. *

* - Use lowercase. + *

+ * Note: These names are used as metric name prefixes. Changing these names will affect + * dashboard/alerts and our public Datadog integration. Please consult the platform teams if unsure. */ @AllArgsConstructor public enum MetricEmittingApps implements MetricEmittingApp { diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricTags.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricTags.java index c067cae7b2d..b630546d36f 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricTags.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricTags.java @@ -21,6 +21,7 @@ public class MetricTags { public static final String AUTHENTICATION_RESPONSE = "authentication_response"; public static final String AUTHENTICATION_RESPONSE_FAILURE_REASON = "authentication_response_failure_reason"; public static final String AUTHENTICATION_REQUEST_URI_ATTRIBUTE_KEY = "request_uri"; + public static final String CANCELLATION_SOURCE = "cancellation_source"; public static final String CONNECTION_ID = "connection_id"; public static final String CRON_TYPE = "cron_type"; public static final String DESTINATION_ID = "destination_id"; @@ -42,6 +43,7 @@ public class MetricTags { public static final String RELEASE_STAGE = "release_stage"; public static final String RESET_WORKFLOW_FAILURE_CAUSE = "failure_cause"; public static final String SOURCE_ID = "source_id"; + public static final String STATUS = "status"; public static final String WORKSPACE_ID = "workspace_id"; public static final String UNKNOWN = "unknown"; public static final String USER_TYPE = "user_type"; // real user, service account, data plane user, etc diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java index 040d26b6821..247f2ddfdbf 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java @@ -28,6 +28,9 @@ * - Add units at name end if applicable. This is especially relevant for time units. *

* - Include the time period in the name if the metric is meant to be run at a certain interval. + *

+ * Note: These names are used as metric name prefixes. Changing these names will affect + * dashboard/alerts and our public Datadog integration. Please consult the platform teams if unsure. */ public enum OssMetricsRegistry implements MetricsRegistry { @@ -339,6 +342,9 @@ public enum OssMetricsRegistry implements MetricsRegistry { WORKFLOWS_HEALED(MetricEmittingApps.CRON, "workflows_healed", "number of workflow the self healing cron healed"), + WORKLOADS_CANCEL(MetricEmittingApps.CRON, + "workload_cancel", + "number of workloads canceled"), NOTIFICATIONS_SENT(MetricEmittingApps.WORKER, "notifications_sent", "number of notifications sent"), diff --git a/airbyte-metrics/reporter/build.gradle b/airbyte-metrics/reporter/build.gradle deleted file mode 100644 index 5d22b5c0ac7..00000000000 --- a/airbyte-metrics/reporter/build.gradle +++ /dev/null @@ -1,53 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.app" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -configurations { - jdbc -} - -configurations.all { - resolutionStrategy { - force libs.jooq - } -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-db:jooq') - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-metrics:metrics-lib') - implementation libs.jooq - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - testImplementation project(':airbyte-test-utils') - testImplementation libs.bundles.micronaut.test - testImplementation libs.postgresql - testImplementation libs.platform.testcontainers.postgresql - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer -} - -airbyte { - application { - name = "airbyte-metrics-reporter" - mainClass = 'io.airbyte.metrics.reporter.Application' - defaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] - } - docker { - imageName = "metrics-reporter" - } -} diff --git a/airbyte-metrics/reporter/build.gradle.kts b/airbyte-metrics/reporter/build.gradle.kts new file mode 100644 index 00000000000..a55f26591e0 --- /dev/null +++ b/airbyte-metrics/reporter/build.gradle.kts @@ -0,0 +1,53 @@ +plugins { + id("io.airbyte.gradle.jvm.app") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +configurations { + create("jdbc") +} + +configurations.all { + resolutionStrategy { + force (libs.jooq) + } +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-db:jooq")) + implementation(project(":airbyte-db:db-lib")) + implementation(project(":airbyte-metrics:metrics-lib")) + implementation(libs.jooq) + + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + + testImplementation(project(":airbyte-test-utils")) + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.postgresql) + testImplementation(libs.platform.testcontainers.postgresql) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) +} + +airbyte { + application { + name = "airbyte-metrics-reporter" + mainClass = "io.airbyte.metrics.reporter.Application" + defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + } + docker { + imageName = "metrics-reporter" + } +} diff --git a/airbyte-micronaut-temporal/build.gradle b/airbyte-micronaut-temporal/build.gradle deleted file mode 100644 index 0d1b70ce4b0..00000000000 --- a/airbyte-micronaut-temporal/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation libs.bundles.micronaut - implementation libs.byte.buddy - implementation libs.guava - implementation libs.spring.core - implementation(libs.temporal.sdk) { - exclude module: 'guava' - } - - implementation project(':airbyte-commons-temporal-core') - - testImplementation libs.assertj.core - testImplementation libs.bundles.junit - testImplementation libs.junit.pioneer - testImplementation libs.mockito.inline - testRuntimeOnly libs.junit.jupiter.engine -} diff --git a/airbyte-micronaut-temporal/build.gradle.kts b/airbyte-micronaut-temporal/build.gradle.kts new file mode 100644 index 00000000000..49b2c9cf2f1 --- /dev/null +++ b/airbyte-micronaut-temporal/build.gradle.kts @@ -0,0 +1,25 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + + implementation(libs.bundles.micronaut) + implementation(libs.byte.buddy) + implementation(libs.guava) + implementation(libs.spring.core) + implementation(libs.temporal.sdk) { + exclude( module = "guava") + } + + implementation(project(":airbyte-commons-temporal-core")) + + testImplementation(libs.assertj.core) + testImplementation(libs.bundles.junit) + testImplementation(libs.junit.pioneer) + testImplementation(libs.mockito.inline) + testRuntimeOnly(libs.junit.jupiter.engine) +} diff --git a/airbyte-notification/build.gradle b/airbyte-notification/build.gradle deleted file mode 100644 index d2d94af05e2..00000000000 --- a/airbyte-notification/build.gradle +++ /dev/null @@ -1,39 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" - id "org.jetbrains.kotlin.jvm" - id "org.jetbrains.kotlin.kapt" -} - -dependencies { - kapt(libs.bundles.micronaut.annotation.processor) - - implementation project(':airbyte-api') - implementation project(':airbyte-commons') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-json-validation') - implementation project(':airbyte-metrics:metrics-lib') - implementation libs.okhttp - implementation 'org.apache.httpcomponents:httpclient:4.5.13' - implementation 'org.commonmark:commonmark:0.21.0' - - compileOnly libs.lombok - annotationProcessor libs.lombok - - implementation libs.guava - implementation libs.bundles.apache - implementation libs.commons.io - implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) - implementation libs.bundles.jackson - // TODO remove this, it's used for String.isEmpty check - implementation libs.bundles.log4j - - testImplementation(libs.mockk) - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer - testImplementation libs.mockito.inline - testImplementation libs.mockwebserver -} diff --git a/airbyte-notification/build.gradle.kts b/airbyte-notification/build.gradle.kts new file mode 100644 index 00000000000..877ef762941 --- /dev/null +++ b/airbyte-notification/build.gradle.kts @@ -0,0 +1,39 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") + id("org.jetbrains.kotlin.jvm") + id("org.jetbrains.kotlin.kapt") +} + +dependencies { + kapt(libs.bundles.micronaut.annotation.processor) + + implementation(project(":airbyte-api")) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-json-validation")) + implementation(project(":airbyte-metrics:metrics-lib")) + implementation(libs.okhttp) + implementation("org.apache.httpcomponents:httpclient:4.5.13") + implementation("org.commonmark:commonmark:0.21.0") + + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + implementation(libs.guava) + implementation(libs.bundles.apache) + implementation(libs.commons.io) + implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) + implementation(libs.bundles.jackson) + // TODO remove this, it"s used for String.isEmpty check) + implementation(libs.bundles.log4j) + + testImplementation(libs.mockk) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) + testImplementation(libs.mockito.inline) + testImplementation(libs.mockwebserver) +} diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java index b3b079bbdf5..5674546d852 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java @@ -291,7 +291,8 @@ boolean sendNotifyRequest(final String urlEndpoint, final String payload) throws LOGGER.info("Successful notification ({}): {}", response.code(), response.body()); return true; } else { - final String errorMessage = String.format("Failed to deliver notification (%s): %s", response.code(), response.body()); + final String body = response.body() != null ? response.body().string() : ""; + final String errorMessage = String.format("Failed to deliver notification (%s): %s", response.code(), body); throw new IOException(errorMessage); } } diff --git a/airbyte-oauth/build.gradle b/airbyte-oauth/build.gradle deleted file mode 100644 index 00f5c558578..00000000000 --- a/airbyte-oauth/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -dependencies { - implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) - implementation libs.bundles.jackson - implementation libs.guava - implementation libs.google.cloud.storage - implementation libs.bundles.apache - implementation libs.appender.log4j2 - implementation libs.aws.java.sdk.s3 - implementation libs.aws.java.sdk.sts - - implementation project(':airbyte-commons') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-json-validation') - implementation libs.airbyte.protocol - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - testImplementation libs.junit.pioneer -} diff --git a/airbyte-oauth/build.gradle.kts b/airbyte-oauth/build.gradle.kts new file mode 100644 index 00000000000..7ad77afa77f --- /dev/null +++ b/airbyte-oauth/build.gradle.kts @@ -0,0 +1,26 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +dependencies { + implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) + implementation(libs.bundles.jackson) + implementation(libs.guava) + implementation(libs.google.cloud.storage) + implementation(libs.bundles.apache) + implementation(libs.appender.log4j2) + implementation(libs.aws.java.sdk.s3) + implementation(libs.aws.java.sdk.sts) + + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-json-validation")) + implementation(libs.airbyte.protocol) + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + testImplementation(libs.junit.pioneer) +} diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java index 02220de0247..f2ffe85aa7d 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java @@ -127,7 +127,9 @@ public Optional createResetConnectionJob(final DestinationConnection desti final List streamsToReset, final UUID workspaceId) throws IOException { - final ConfiguredAirbyteCatalog configuredAirbyteCatalog = standardSync.getCatalog(); + final ConfiguredAirbyteCatalog immutableConfiguredAirbyteCatalog = standardSync.getCatalog(); + final ConfiguredAirbyteCatalog configuredAirbyteCatalog = new ConfiguredAirbyteCatalog() + .withStreams(new ArrayList<>(immutableConfiguredAirbyteCatalog.getStreams())); CatalogTransforms.updateCatalogForReset(streamsToReset, configuredAirbyteCatalog); final var resetResourceRequirements = diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java index 239b67e7652..ef8364b47f8 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java @@ -86,11 +86,6 @@ */ public class DefaultJobPersistence implements JobPersistence { - // not static because job history test case manipulates these. - private final int jobHistoryMinimumAgeInDays; - private final int jobHistoryMinimumRecency; - private final int jobHistoryExcessiveNumberOfJobs; - private static final Logger LOGGER = LoggerFactory.getLogger(DefaultJobPersistence.class); private static final String ATTEMPT_NUMBER = "attempt_number"; private static final String JOB_ID = "job_id"; @@ -100,10 +95,6 @@ public class DefaultJobPersistence implements JobPersistence { private static final String DEPLOYMENT_ID_KEY = "deployment_id"; private static final String METADATA_KEY_COL = "key"; private static final String METADATA_VAL_COL = "value"; - - @VisibleForTesting - static final String BASE_JOB_SELECT_AND_JOIN = jobSelectAndJoin("jobs"); - private static final String AIRBYTE_METADATA_TABLE = "airbyte_metadata"; private static final String ORDER_BY_JOB_TIME_ATTEMPT_TIME = "ORDER BY jobs.created_at DESC, jobs.id DESC, attempts.created_at ASC, attempts.id ASC "; @@ -114,55 +105,393 @@ public class DefaultJobPersistence implements JobPersistence { .map(DefaultJobPersistence::toSqlName) .map(Names::singleQuote) .collect(Collectors.joining(","))); + private static final String ATTEMPT_FIELDS = """ + attempts.attempt_number AS attempt_number, + attempts.attempt_sync_config AS attempt_sync_config, + attempts.log_path AS log_path, + attempts.output AS attempt_output, + attempts.status AS attempt_status, + attempts.processing_task_queue AS processing_task_queue, + attempts.failure_summary AS attempt_failure_summary, + attempts.created_at AS attempt_created_at, + attempts.updated_at AS attempt_updated_at, + attempts.ended_at AS attempt_ended_at + """; + @VisibleForTesting + static final String BASE_JOB_SELECT_AND_JOIN = jobSelectAndJoin("jobs"); + private static final String ATTEMPT_SELECT = + "SELECT job_id," + ATTEMPT_FIELDS + "FROM attempts WHERE job_id = ? AND attempt_number = ?"; + // not static because job history test case manipulates these. + private final int jobHistoryMinimumAgeInDays; + private final int jobHistoryMinimumRecency; + private final int jobHistoryExcessiveNumberOfJobs; + private final ExceptionWrappingDatabase jobDatabase; + private final Supplier timeSupplier; + + @VisibleForTesting + DefaultJobPersistence(final Database jobDatabase, + final Supplier timeSupplier, + final int minimumAgeInDays, + final int excessiveNumberOfJobs, + final int minimumRecencyCount) { + this.jobDatabase = new ExceptionWrappingDatabase(jobDatabase); + this.timeSupplier = timeSupplier; + jobHistoryMinimumAgeInDays = minimumAgeInDays; + jobHistoryExcessiveNumberOfJobs = excessiveNumberOfJobs; + jobHistoryMinimumRecency = minimumRecencyCount; + } + + public DefaultJobPersistence(final Database jobDatabase) { + this(jobDatabase, Instant::now, 30, 500, 10); + } + + private static String jobSelectAndJoin(final String jobsSubquery) { + return "SELECT\n" + + "jobs.id AS job_id,\n" + + "jobs.config_type AS config_type,\n" + + "jobs.scope AS scope,\n" + + "jobs.config AS config,\n" + + "jobs.status AS job_status,\n" + + "jobs.started_at AS job_started_at,\n" + + "jobs.created_at AS job_created_at,\n" + + "jobs.updated_at AS job_updated_at,\n" + + ATTEMPT_FIELDS + + "FROM " + jobsSubquery + " LEFT OUTER JOIN attempts ON jobs.id = attempts.job_id "; + } + + private static void saveToSyncStatsTable(final OffsetDateTime now, final SyncStats syncStats, final Long attemptId, final DSLContext ctx) { + // Although JOOQ supports upsert using the onConflict statement, we cannot use it as the table + // currently has duplicate records and also doesn't contain the unique constraint on the attempt_id + // column JOOQ requires. We are forced to check for existence. + final var isExisting = ctx.fetchExists(SYNC_STATS, SYNC_STATS.ATTEMPT_ID.eq(attemptId)); + if (isExisting) { + ctx.update(SYNC_STATS) + .set(SYNC_STATS.UPDATED_AT, now) + .set(SYNC_STATS.BYTES_EMITTED, syncStats.getBytesEmitted()) + .set(SYNC_STATS.RECORDS_EMITTED, syncStats.getRecordsEmitted()) + .set(SYNC_STATS.ESTIMATED_RECORDS, syncStats.getEstimatedRecords()) + .set(SYNC_STATS.ESTIMATED_BYTES, syncStats.getEstimatedBytes()) + .set(SYNC_STATS.RECORDS_COMMITTED, syncStats.getRecordsCommitted()) + .set(SYNC_STATS.BYTES_COMMITTED, syncStats.getBytesCommitted()) + .set(SYNC_STATS.SOURCE_STATE_MESSAGES_EMITTED, syncStats.getSourceStateMessagesEmitted()) + .set(SYNC_STATS.DESTINATION_STATE_MESSAGES_EMITTED, syncStats.getDestinationStateMessagesEmitted()) + .set(SYNC_STATS.MAX_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED, syncStats.getMaxSecondsBeforeSourceStateMessageEmitted()) + .set(SYNC_STATS.MEAN_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED, syncStats.getMeanSecondsBeforeSourceStateMessageEmitted()) + .set(SYNC_STATS.MAX_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED, syncStats.getMaxSecondsBetweenStateMessageEmittedandCommitted()) + .set(SYNC_STATS.MEAN_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED, syncStats.getMeanSecondsBetweenStateMessageEmittedandCommitted()) + .where(SYNC_STATS.ATTEMPT_ID.eq(attemptId)) + .execute(); + return; + } + + ctx.insertInto(SYNC_STATS) + .set(SYNC_STATS.ID, UUID.randomUUID()) + .set(SYNC_STATS.CREATED_AT, now) + .set(SYNC_STATS.ATTEMPT_ID, attemptId) + .set(SYNC_STATS.UPDATED_AT, now) + .set(SYNC_STATS.BYTES_EMITTED, syncStats.getBytesEmitted()) + .set(SYNC_STATS.RECORDS_EMITTED, syncStats.getRecordsEmitted()) + .set(SYNC_STATS.ESTIMATED_RECORDS, syncStats.getEstimatedRecords()) + .set(SYNC_STATS.ESTIMATED_BYTES, syncStats.getEstimatedBytes()) + .set(SYNC_STATS.RECORDS_COMMITTED, syncStats.getRecordsCommitted()) + .set(SYNC_STATS.BYTES_COMMITTED, syncStats.getBytesCommitted()) + .set(SYNC_STATS.SOURCE_STATE_MESSAGES_EMITTED, syncStats.getSourceStateMessagesEmitted()) + .set(SYNC_STATS.DESTINATION_STATE_MESSAGES_EMITTED, syncStats.getDestinationStateMessagesEmitted()) + .set(SYNC_STATS.MAX_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED, syncStats.getMaxSecondsBeforeSourceStateMessageEmitted()) + .set(SYNC_STATS.MEAN_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED, syncStats.getMeanSecondsBeforeSourceStateMessageEmitted()) + .set(SYNC_STATS.MAX_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED, syncStats.getMaxSecondsBetweenStateMessageEmittedandCommitted()) + .set(SYNC_STATS.MEAN_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED, syncStats.getMeanSecondsBetweenStateMessageEmittedandCommitted()) + .execute(); + } + + private static void saveToStreamStatsTableBatch(final OffsetDateTime now, + final List perStreamStats, + final Long attemptId, + final DSLContext ctx) { + final List queries = new ArrayList<>(); + + // Upserts require the onConflict statement that does not work as the table currently has duplicate + // records on the null + // namespace value. This is a valid state and not a bug. + // Upserts are possible if we upgrade to Postgres 15. However this requires downtime. A simpler + // solution to prevent O(N) existence checks, where N in the + // number of streams, is to fetch all streams for the attempt. Existence checks are in memory, + // letting us do only 2 queries in total. + final Set existingStreams = ctx.select(STREAM_STATS.STREAM_NAME, STREAM_STATS.STREAM_NAMESPACE) + .from(STREAM_STATS) + .where(STREAM_STATS.ATTEMPT_ID.eq(attemptId)) + .fetchSet(r -> new StreamDescriptor().withName(r.get(STREAM_STATS.STREAM_NAME)).withNamespace(r.get(STREAM_STATS.STREAM_NAMESPACE))); + + Optional.ofNullable(perStreamStats).orElse(Collections.emptyList()).forEach( + streamStats -> { + final var isExisting = + existingStreams.contains(new StreamDescriptor().withName(streamStats.getStreamName()).withNamespace(streamStats.getStreamNamespace())); + final var stats = streamStats.getStats(); + if (isExisting) { + queries.add( + ctx.update(STREAM_STATS) + .set(STREAM_STATS.UPDATED_AT, now) + .set(STREAM_STATS.BYTES_EMITTED, stats.getBytesEmitted()) + .set(STREAM_STATS.RECORDS_EMITTED, stats.getRecordsEmitted()) + .set(STREAM_STATS.ESTIMATED_RECORDS, stats.getEstimatedRecords()) + .set(STREAM_STATS.ESTIMATED_BYTES, stats.getEstimatedBytes()) + .set(STREAM_STATS.BYTES_COMMITTED, stats.getBytesCommitted()) + .set(STREAM_STATS.RECORDS_COMMITTED, stats.getRecordsCommitted()) + .where( + STREAM_STATS.ATTEMPT_ID.eq(attemptId), + PersistenceHelpers.isNullOrEquals(STREAM_STATS.STREAM_NAME, streamStats.getStreamName()), + PersistenceHelpers.isNullOrEquals(STREAM_STATS.STREAM_NAMESPACE, streamStats.getStreamNamespace()))); + } else { + queries.add( + ctx.insertInto(STREAM_STATS) + .set(STREAM_STATS.ID, UUID.randomUUID()) + .set(STREAM_STATS.ATTEMPT_ID, attemptId) + .set(STREAM_STATS.STREAM_NAME, streamStats.getStreamName()) + .set(STREAM_STATS.STREAM_NAMESPACE, streamStats.getStreamNamespace()) + .set(STREAM_STATS.CREATED_AT, now) + .set(STREAM_STATS.UPDATED_AT, now) + .set(STREAM_STATS.BYTES_EMITTED, stats.getBytesEmitted()) + .set(STREAM_STATS.RECORDS_EMITTED, stats.getRecordsEmitted()) + .set(STREAM_STATS.ESTIMATED_RECORDS, stats.getEstimatedRecords()) + .set(STREAM_STATS.ESTIMATED_BYTES, stats.getEstimatedBytes()) + .set(STREAM_STATS.BYTES_COMMITTED, stats.getBytesCommitted()) + .set(STREAM_STATS.RECORDS_COMMITTED, stats.getRecordsCommitted())); + } + }); + + ctx.batch(queries).execute(); + } + + private static Map hydrateSyncStats(final String jobIdsStr, final DSLContext ctx) { + final var attemptStats = new HashMap(); + final var syncResults = ctx.fetch( + "SELECT atmpt.attempt_number, atmpt.job_id," + + "stats.estimated_bytes, stats.estimated_records, stats.bytes_emitted, stats.records_emitted, " + + "stats.bytes_committed, stats.records_committed " + + "FROM sync_stats stats " + + "INNER JOIN attempts atmpt ON stats.attempt_id = atmpt.id " + + "WHERE job_id IN ( " + jobIdsStr + ");"); + syncResults.forEach(r -> { + final var key = new JobAttemptPair(r.get(ATTEMPTS.JOB_ID), r.get(ATTEMPTS.ATTEMPT_NUMBER)); + final var syncStats = new SyncStats() + .withBytesEmitted(r.get(SYNC_STATS.BYTES_EMITTED)) + .withRecordsEmitted(r.get(SYNC_STATS.RECORDS_EMITTED)) + .withEstimatedRecords(r.get(SYNC_STATS.ESTIMATED_RECORDS)) + .withEstimatedBytes(r.get(SYNC_STATS.ESTIMATED_BYTES)) + .withBytesCommitted(r.get(SYNC_STATS.BYTES_COMMITTED)) + .withRecordsCommitted(r.get(SYNC_STATS.RECORDS_COMMITTED)); + attemptStats.put(key, new AttemptStats(syncStats, Lists.newArrayList())); + }); + return attemptStats; + } + + /** + * This method needed to be called after + * {@link DefaultJobPersistence#hydrateSyncStats(String, DSLContext)} as it assumes hydrateSyncStats + * has prepopulated the map. + */ + private static void hydrateStreamStats(final String jobIdsStr, final DSLContext ctx, final Map attemptStats) { + final var streamResults = ctx.fetch( + "SELECT atmpt.attempt_number, atmpt.job_id, " + + "stats.stream_name, stats.stream_namespace, stats.estimated_bytes, stats.estimated_records, stats.bytes_emitted, stats.records_emitted," + + "stats.bytes_committed, stats.records_committed " + + "FROM stream_stats stats " + + "INNER JOIN attempts atmpt ON atmpt.id = stats.attempt_id " + + "WHERE attempt_id IN " + + "( SELECT id FROM attempts WHERE job_id IN ( " + jobIdsStr + "));"); + + streamResults.forEach(r -> { + final var streamSyncStats = new StreamSyncStats() + .withStreamNamespace(r.get(STREAM_STATS.STREAM_NAMESPACE)) + .withStreamName(r.get(STREAM_STATS.STREAM_NAME)) + .withStats(new SyncStats() + .withBytesEmitted(r.get(STREAM_STATS.BYTES_EMITTED)) + .withRecordsEmitted(r.get(STREAM_STATS.RECORDS_EMITTED)) + .withEstimatedRecords(r.get(STREAM_STATS.ESTIMATED_RECORDS)) + .withEstimatedBytes(r.get(STREAM_STATS.ESTIMATED_BYTES)) + .withBytesCommitted(r.get(STREAM_STATS.BYTES_COMMITTED)) + .withRecordsCommitted(r.get(STREAM_STATS.RECORDS_COMMITTED))); + + final var key = new JobAttemptPair(r.get(ATTEMPTS.JOB_ID), r.get(ATTEMPTS.ATTEMPT_NUMBER)); + if (!attemptStats.containsKey(key)) { + LOGGER.error("{} stream stats entry does not have a corresponding sync stats entry. This suggest the database is in a bad state.", key); + return; + } + attemptStats.get(key).perStreamStats().add(streamSyncStats); + }); + } + + @VisibleForTesting + static Long getAttemptId(final long jobId, final int attemptNumber, final DSLContext ctx) { + final Optional record = + ctx.fetch("SELECT id from attempts where job_id = ? AND attempt_number = ?", jobId, + attemptNumber).stream().findFirst(); + if (record.isEmpty()) { + return -1L; + } + + return record.get().get("id", Long.class); + } + + private static RecordMapper getSyncStatsRecordMapper() { + return record -> new SyncStats().withBytesEmitted(record.get(SYNC_STATS.BYTES_EMITTED)).withRecordsEmitted(record.get(SYNC_STATS.RECORDS_EMITTED)) + .withEstimatedBytes(record.get(SYNC_STATS.ESTIMATED_BYTES)).withEstimatedRecords(record.get(SYNC_STATS.ESTIMATED_RECORDS)) + .withSourceStateMessagesEmitted(record.get(SYNC_STATS.SOURCE_STATE_MESSAGES_EMITTED)) + .withDestinationStateMessagesEmitted(record.get(SYNC_STATS.DESTINATION_STATE_MESSAGES_EMITTED)) + .withBytesCommitted(record.get(SYNC_STATS.BYTES_COMMITTED)) + .withRecordsCommitted(record.get(SYNC_STATS.RECORDS_COMMITTED)) + .withMeanSecondsBeforeSourceStateMessageEmitted(record.get(SYNC_STATS.MEAN_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED)) + .withMaxSecondsBeforeSourceStateMessageEmitted(record.get(SYNC_STATS.MAX_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED)) + .withMeanSecondsBetweenStateMessageEmittedandCommitted(record.get(SYNC_STATS.MEAN_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED)) + .withMaxSecondsBetweenStateMessageEmittedandCommitted(record.get(SYNC_STATS.MAX_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED)); + } + + private static RecordMapper getStreamStatsRecordsMapper() { + return record -> { + final var stats = new SyncStats() + .withEstimatedRecords(record.get(STREAM_STATS.ESTIMATED_RECORDS)).withEstimatedBytes(record.get(STREAM_STATS.ESTIMATED_BYTES)) + .withRecordsEmitted(record.get(STREAM_STATS.RECORDS_EMITTED)).withBytesEmitted(record.get(STREAM_STATS.BYTES_EMITTED)) + .withRecordsCommitted(record.get(STREAM_STATS.RECORDS_COMMITTED)).withBytesCommitted(record.get(STREAM_STATS.BYTES_COMMITTED)); + return new StreamSyncStats() + .withStreamName(record.get(STREAM_STATS.STREAM_NAME)).withStreamNamespace(record.get(STREAM_STATS.STREAM_NAMESPACE)) + .withStats(stats); + }; + } + + private static RecordMapper getNormalizationSummaryRecordMapper() { + return record -> { + try { + return new NormalizationSummary().withStartTime(record.get(NORMALIZATION_SUMMARIES.START_TIME).toInstant().toEpochMilli()) + .withEndTime(record.get(NORMALIZATION_SUMMARIES.END_TIME).toInstant().toEpochMilli()) + .withFailures(record.get(NORMALIZATION_SUMMARIES.FAILURES, String.class) == null ? null : deserializeFailureReasons(record)); + } catch (final JsonProcessingException e) { + throw new RuntimeException(e); + } + }; + } + + private static List deserializeFailureReasons(final Record record) throws JsonProcessingException { + final ObjectMapper mapper = new ObjectMapper(); + return List.of(mapper.readValue(String.valueOf(record.get(NORMALIZATION_SUMMARIES.FAILURES)), FailureReason[].class)); + } + + // Retrieves only Job information from the record, without any attempt info + private static Job getJobFromRecord(final Record record) { + return new Job(record.get(JOB_ID, Long.class), + Enums.toEnum(record.get("config_type", String.class), ConfigType.class).orElseThrow(), + record.get("scope", String.class), + parseJobConfigFromString(record.get("config", String.class)), + new ArrayList(), + JobStatus.valueOf(record.get("job_status", String.class).toUpperCase()), + Optional.ofNullable(record.get("job_started_at")).map(value -> getEpoch(record, "started_at")).orElse(null), + getEpoch(record, "job_created_at"), + getEpoch(record, "job_updated_at")); + } + + private static JobConfig parseJobConfigFromString(final String jobConfigString) { + final JobConfig jobConfig = Jsons.deserialize(jobConfigString, JobConfig.class); + // On-the-fly migration of persisted data types related objects (protocol v0->v1) + if (jobConfig.getConfigType() == ConfigType.SYNC && jobConfig.getSync() != null) { + // TODO feature flag this for data types rollout + // CatalogMigrationV1Helper.upgradeSchemaIfNeeded(jobConfig.getSync().getConfiguredAirbyteCatalog()); + CatalogMigrationV1Helper.downgradeSchemaIfNeeded(jobConfig.getSync().getConfiguredAirbyteCatalog()); + } else if (jobConfig.getConfigType() == ConfigType.RESET_CONNECTION && jobConfig.getResetConnection() != null) { + // TODO feature flag this for data types rollout + // CatalogMigrationV1Helper.upgradeSchemaIfNeeded(jobConfig.getResetConnection().getConfiguredAirbyteCatalog()); + CatalogMigrationV1Helper.downgradeSchemaIfNeeded(jobConfig.getResetConnection().getConfiguredAirbyteCatalog()); + } + return jobConfig; + } + + private static Attempt getAttemptFromRecord(final Record record) { + final String attemptOutputString = record.get("attempt_output", String.class); + return new Attempt( + record.get(ATTEMPT_NUMBER, int.class), + record.get(JOB_ID, Long.class), + Path.of(record.get("log_path", String.class)), + record.get("attempt_sync_config", String.class) == null ? null + : Jsons.deserialize(record.get("attempt_sync_config", String.class), AttemptSyncConfig.class), + attemptOutputString == null ? null : parseJobOutputFromString(attemptOutputString), + Enums.toEnum(record.get("attempt_status", String.class), AttemptStatus.class).orElseThrow(), + record.get("processing_task_queue", String.class), + record.get("attempt_failure_summary", String.class) == null ? null + : Jsons.deserialize(record.get("attempt_failure_summary", String.class), AttemptFailureSummary.class), + getEpoch(record, "attempt_created_at"), + getEpoch(record, "attempt_updated_at"), + Optional.ofNullable(record.get("attempt_ended_at")) + .map(value -> getEpoch(record, "attempt_ended_at")) + .orElse(null)); + } + + private static JobOutput parseJobOutputFromString(final String jobOutputString) { + final JobOutput jobOutput = Jsons.deserialize(jobOutputString, JobOutput.class); + // On-the-fly migration of persisted data types related objects (protocol v0->v1) + if (jobOutput.getOutputType() == OutputType.DISCOVER_CATALOG && jobOutput.getDiscoverCatalog() != null) { + // TODO feature flag this for data types rollout + // CatalogMigrationV1Helper.upgradeSchemaIfNeeded(jobOutput.getDiscoverCatalog().getCatalog()); + CatalogMigrationV1Helper.downgradeSchemaIfNeeded(jobOutput.getDiscoverCatalog().getCatalog()); + } else if (jobOutput.getOutputType() == OutputType.SYNC && jobOutput.getSync() != null) { + // TODO feature flag this for data types rollout + // CatalogMigrationV1Helper.upgradeSchemaIfNeeded(jobOutput.getSync().getOutputCatalog()); + CatalogMigrationV1Helper.downgradeSchemaIfNeeded(jobOutput.getSync().getOutputCatalog()); + } + return jobOutput; + } + + private static List getAttemptsWithJobsFromResult(final Result result) { + return result + .stream() + .filter(record -> record.getValue(ATTEMPT_NUMBER) != null) + .map(record -> new AttemptWithJobInfo(getAttemptFromRecord(record), getJobFromRecord(record))) + .collect(Collectors.toList()); + } - private static final String ATTEMPT_FIELDS = """ - attempts.attempt_number AS attempt_number, - attempts.attempt_sync_config AS attempt_sync_config, - attempts.log_path AS log_path, - attempts.output AS attempt_output, - attempts.status AS attempt_status, - attempts.processing_task_queue AS processing_task_queue, - attempts.failure_summary AS attempt_failure_summary, - attempts.created_at AS attempt_created_at, - attempts.updated_at AS attempt_updated_at, - attempts.ended_at AS attempt_ended_at - """; + private static List getJobsFromResult(final Result result) { + // keeps results strictly in order so the sql query controls the sort + final List jobs = new ArrayList<>(); + Job currentJob = null; + for (final Record entry : result) { + if (currentJob == null || currentJob.getId() != entry.get(JOB_ID, Long.class)) { + currentJob = getJobFromRecord(entry); + jobs.add(currentJob); + } + if (entry.getValue(ATTEMPT_NUMBER) != null) { + currentJob.getAttempts().add(getAttemptFromRecord(entry)); + } + } - private static final String ATTEMPT_SELECT = - "SELECT job_id," + ATTEMPT_FIELDS + "FROM attempts WHERE job_id = ? AND attempt_number = ?"; + return jobs; + } - private final ExceptionWrappingDatabase jobDatabase; - private final Supplier timeSupplier; + /** + * Generate a string fragment that can be put in the IN clause of a SQL statement. eg. column IN + * (value1, value2) + * + * @param values to encode + * @param enum type + * @return "'value1', 'value2', 'value3'" + */ + private static > String toSqlInFragment(final Iterable values) { + return StreamSupport.stream(values.spliterator(), false).map(DefaultJobPersistence::toSqlName).map(Names::singleQuote) + .collect(Collectors.joining(",", "(", ")")); + } @VisibleForTesting - DefaultJobPersistence(final Database jobDatabase, - final Supplier timeSupplier, - final int minimumAgeInDays, - final int excessiveNumberOfJobs, - final int minimumRecencyCount) { - this.jobDatabase = new ExceptionWrappingDatabase(jobDatabase); - this.timeSupplier = timeSupplier; - jobHistoryMinimumAgeInDays = minimumAgeInDays; - jobHistoryExcessiveNumberOfJobs = excessiveNumberOfJobs; - jobHistoryMinimumRecency = minimumRecencyCount; + static > String toSqlName(final T value) { + return value.name().toLowerCase(); } - public DefaultJobPersistence(final Database jobDatabase) { - this(jobDatabase, Instant::now, 30, 500, 10); + private static > Set configTypeSqlNames(final Set configTypes) { + return configTypes.stream().map(DefaultJobPersistence::toSqlName).collect(Collectors.toSet()); } - private static String jobSelectAndJoin(final String jobsSubquery) { - return "SELECT\n" - + "jobs.id AS job_id,\n" - + "jobs.config_type AS config_type,\n" - + "jobs.scope AS scope,\n" - + "jobs.config AS config,\n" - + "jobs.status AS job_status,\n" - + "jobs.started_at AS job_started_at,\n" - + "jobs.created_at AS job_created_at,\n" - + "jobs.updated_at AS job_updated_at,\n" - + ATTEMPT_FIELDS - + "FROM " + jobsSubquery + " LEFT OUTER JOIN attempts ON jobs.id = attempts.job_id "; + @VisibleForTesting + static Optional getJobFromResult(final Result result) { + return getJobsFromResult(result).stream().findFirst(); + } + + private static long getEpoch(final Record record, final String fieldName) { + return record.get(fieldName, LocalDateTime.class).toEpochSecond(ZoneOffset.UTC); } /** @@ -429,109 +758,6 @@ public void writeStats(final long jobId, } - private static void saveToSyncStatsTable(final OffsetDateTime now, final SyncStats syncStats, final Long attemptId, final DSLContext ctx) { - // Although JOOQ supports upsert using the onConflict statement, we cannot use it as the table - // currently has duplicate records and also doesn't contain the unique constraint on the attempt_id - // column JOOQ requires. We are forced to check for existence. - final var isExisting = ctx.fetchExists(SYNC_STATS, SYNC_STATS.ATTEMPT_ID.eq(attemptId)); - if (isExisting) { - ctx.update(SYNC_STATS) - .set(SYNC_STATS.UPDATED_AT, now) - .set(SYNC_STATS.BYTES_EMITTED, syncStats.getBytesEmitted()) - .set(SYNC_STATS.RECORDS_EMITTED, syncStats.getRecordsEmitted()) - .set(SYNC_STATS.ESTIMATED_RECORDS, syncStats.getEstimatedRecords()) - .set(SYNC_STATS.ESTIMATED_BYTES, syncStats.getEstimatedBytes()) - .set(SYNC_STATS.RECORDS_COMMITTED, syncStats.getRecordsCommitted()) - .set(SYNC_STATS.BYTES_COMMITTED, syncStats.getBytesCommitted()) - .set(SYNC_STATS.SOURCE_STATE_MESSAGES_EMITTED, syncStats.getSourceStateMessagesEmitted()) - .set(SYNC_STATS.DESTINATION_STATE_MESSAGES_EMITTED, syncStats.getDestinationStateMessagesEmitted()) - .set(SYNC_STATS.MAX_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED, syncStats.getMaxSecondsBeforeSourceStateMessageEmitted()) - .set(SYNC_STATS.MEAN_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED, syncStats.getMeanSecondsBeforeSourceStateMessageEmitted()) - .set(SYNC_STATS.MAX_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED, syncStats.getMaxSecondsBetweenStateMessageEmittedandCommitted()) - .set(SYNC_STATS.MEAN_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED, syncStats.getMeanSecondsBetweenStateMessageEmittedandCommitted()) - .where(SYNC_STATS.ATTEMPT_ID.eq(attemptId)) - .execute(); - return; - } - - ctx.insertInto(SYNC_STATS) - .set(SYNC_STATS.ID, UUID.randomUUID()) - .set(SYNC_STATS.CREATED_AT, now) - .set(SYNC_STATS.ATTEMPT_ID, attemptId) - .set(SYNC_STATS.UPDATED_AT, now) - .set(SYNC_STATS.BYTES_EMITTED, syncStats.getBytesEmitted()) - .set(SYNC_STATS.RECORDS_EMITTED, syncStats.getRecordsEmitted()) - .set(SYNC_STATS.ESTIMATED_RECORDS, syncStats.getEstimatedRecords()) - .set(SYNC_STATS.ESTIMATED_BYTES, syncStats.getEstimatedBytes()) - .set(SYNC_STATS.RECORDS_COMMITTED, syncStats.getRecordsCommitted()) - .set(SYNC_STATS.BYTES_COMMITTED, syncStats.getBytesCommitted()) - .set(SYNC_STATS.SOURCE_STATE_MESSAGES_EMITTED, syncStats.getSourceStateMessagesEmitted()) - .set(SYNC_STATS.DESTINATION_STATE_MESSAGES_EMITTED, syncStats.getDestinationStateMessagesEmitted()) - .set(SYNC_STATS.MAX_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED, syncStats.getMaxSecondsBeforeSourceStateMessageEmitted()) - .set(SYNC_STATS.MEAN_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED, syncStats.getMeanSecondsBeforeSourceStateMessageEmitted()) - .set(SYNC_STATS.MAX_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED, syncStats.getMaxSecondsBetweenStateMessageEmittedandCommitted()) - .set(SYNC_STATS.MEAN_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED, syncStats.getMeanSecondsBetweenStateMessageEmittedandCommitted()) - .execute(); - } - - private static void saveToStreamStatsTableBatch(final OffsetDateTime now, - final List perStreamStats, - final Long attemptId, - final DSLContext ctx) { - final List queries = new ArrayList<>(); - - // Upserts require the onConflict statement that does not work as the table currently has duplicate - // records on the null - // namespace value. This is a valid state and not a bug. - // Upserts are possible if we upgrade to Postgres 15. However this requires downtime. A simpler - // solution to prevent O(N) existence checks, where N in the - // number of streams, is to fetch all streams for the attempt. Existence checks are in memory, - // letting us do only 2 queries in total. - final Set existingStreams = ctx.select(STREAM_STATS.STREAM_NAME, STREAM_STATS.STREAM_NAMESPACE) - .from(STREAM_STATS) - .where(STREAM_STATS.ATTEMPT_ID.eq(attemptId)) - .fetchSet(r -> new StreamDescriptor().withName(r.get(STREAM_STATS.STREAM_NAME)).withNamespace(r.get(STREAM_STATS.STREAM_NAMESPACE))); - - Optional.ofNullable(perStreamStats).orElse(Collections.emptyList()).forEach( - streamStats -> { - final var isExisting = - existingStreams.contains(new StreamDescriptor().withName(streamStats.getStreamName()).withNamespace(streamStats.getStreamNamespace())); - final var stats = streamStats.getStats(); - if (isExisting) { - queries.add( - ctx.update(STREAM_STATS) - .set(STREAM_STATS.UPDATED_AT, now) - .set(STREAM_STATS.BYTES_EMITTED, stats.getBytesEmitted()) - .set(STREAM_STATS.RECORDS_EMITTED, stats.getRecordsEmitted()) - .set(STREAM_STATS.ESTIMATED_RECORDS, stats.getEstimatedRecords()) - .set(STREAM_STATS.ESTIMATED_BYTES, stats.getEstimatedBytes()) - .set(STREAM_STATS.BYTES_COMMITTED, stats.getBytesCommitted()) - .set(STREAM_STATS.RECORDS_COMMITTED, stats.getRecordsCommitted()) - .where( - STREAM_STATS.ATTEMPT_ID.eq(attemptId), - PersistenceHelpers.isNullOrEquals(STREAM_STATS.STREAM_NAME, streamStats.getStreamName()), - PersistenceHelpers.isNullOrEquals(STREAM_STATS.STREAM_NAMESPACE, streamStats.getStreamNamespace()))); - } else { - queries.add( - ctx.insertInto(STREAM_STATS) - .set(STREAM_STATS.ID, UUID.randomUUID()) - .set(STREAM_STATS.ATTEMPT_ID, attemptId) - .set(STREAM_STATS.STREAM_NAME, streamStats.getStreamName()) - .set(STREAM_STATS.STREAM_NAMESPACE, streamStats.getStreamNamespace()) - .set(STREAM_STATS.CREATED_AT, now) - .set(STREAM_STATS.UPDATED_AT, now) - .set(STREAM_STATS.BYTES_EMITTED, stats.getBytesEmitted()) - .set(STREAM_STATS.RECORDS_EMITTED, stats.getRecordsEmitted()) - .set(STREAM_STATS.ESTIMATED_RECORDS, stats.getEstimatedRecords()) - .set(STREAM_STATS.ESTIMATED_BYTES, stats.getEstimatedBytes()) - .set(STREAM_STATS.BYTES_COMMITTED, stats.getBytesCommitted()) - .set(STREAM_STATS.RECORDS_COMMITTED, stats.getRecordsCommitted())); - } - }); - - ctx.batch(queries).execute(); - } - @Override public void writeAttemptSyncConfig(final long jobId, final int attemptNumber, final AttemptSyncConfig attemptSyncConfig) throws IOException { final OffsetDateTime now = OffsetDateTime.ofInstant(timeSupplier.get(), ZoneOffset.UTC); @@ -573,90 +799,31 @@ public AttemptStats getAttemptStats(final long jobId, final int attemptNumber) t @Override public Map getAttemptStats(final List jobIds) throws IOException { if (jobIds == null || jobIds.isEmpty()) { - return Map.of(); - } - - final var jobIdsStr = StringUtils.join(jobIds, ','); - return jobDatabase.query(ctx -> { - // Instead of one massive join query, separate this query into two queries for better readability - // for now. - // We can combine the queries at a later date if this still proves to be not efficient enough. - final Map attemptStats = hydrateSyncStats(jobIdsStr, ctx); - hydrateStreamStats(jobIdsStr, ctx, attemptStats); - return attemptStats; - }); - } - - @Override - public SyncStats getAttemptCombinedStats(final long jobId, final int attemptNumber) throws IOException { - return jobDatabase - .query(ctx -> { - final Long attemptId = getAttemptId(jobId, attemptNumber, ctx); - return ctx.select(DSL.asterisk()).from(SYNC_STATS).where(SYNC_STATS.ATTEMPT_ID.eq(attemptId)) - .orderBy(SYNC_STATS.UPDATED_AT.desc()) - .fetchOne(getSyncStatsRecordMapper()); - }); - } - - private static Map hydrateSyncStats(final String jobIdsStr, final DSLContext ctx) { - final var attemptStats = new HashMap(); - final var syncResults = ctx.fetch( - "SELECT atmpt.attempt_number, atmpt.job_id," - + "stats.estimated_bytes, stats.estimated_records, stats.bytes_emitted, stats.records_emitted, " - + "stats.bytes_committed, stats.records_committed " - + "FROM sync_stats stats " - + "INNER JOIN attempts atmpt ON stats.attempt_id = atmpt.id " - + "WHERE job_id IN ( " + jobIdsStr + ");"); - syncResults.forEach(r -> { - final var key = new JobAttemptPair(r.get(ATTEMPTS.JOB_ID), r.get(ATTEMPTS.ATTEMPT_NUMBER)); - final var syncStats = new SyncStats() - .withBytesEmitted(r.get(SYNC_STATS.BYTES_EMITTED)) - .withRecordsEmitted(r.get(SYNC_STATS.RECORDS_EMITTED)) - .withEstimatedRecords(r.get(SYNC_STATS.ESTIMATED_RECORDS)) - .withEstimatedBytes(r.get(SYNC_STATS.ESTIMATED_BYTES)) - .withBytesCommitted(r.get(SYNC_STATS.BYTES_COMMITTED)) - .withRecordsCommitted(r.get(SYNC_STATS.RECORDS_COMMITTED)); - attemptStats.put(key, new AttemptStats(syncStats, Lists.newArrayList())); - }); - return attemptStats; - } - - /** - * This method needed to be called after - * {@link DefaultJobPersistence#hydrateSyncStats(String, DSLContext)} as it assumes hydrateSyncStats - * has prepopulated the map. - */ - private static void hydrateStreamStats(final String jobIdsStr, final DSLContext ctx, final Map attemptStats) { - final var streamResults = ctx.fetch( - "SELECT atmpt.attempt_number, atmpt.job_id, " - + "stats.stream_name, stats.stream_namespace, stats.estimated_bytes, stats.estimated_records, stats.bytes_emitted, stats.records_emitted," - + "stats.bytes_committed, stats.records_committed " - + "FROM stream_stats stats " - + "INNER JOIN attempts atmpt ON atmpt.id = stats.attempt_id " - + "WHERE attempt_id IN " - + "( SELECT id FROM attempts WHERE job_id IN ( " + jobIdsStr + "));"); - - streamResults.forEach(r -> { - final var streamSyncStats = new StreamSyncStats() - .withStreamNamespace(r.get(STREAM_STATS.STREAM_NAMESPACE)) - .withStreamName(r.get(STREAM_STATS.STREAM_NAME)) - .withStats(new SyncStats() - .withBytesEmitted(r.get(STREAM_STATS.BYTES_EMITTED)) - .withRecordsEmitted(r.get(STREAM_STATS.RECORDS_EMITTED)) - .withEstimatedRecords(r.get(STREAM_STATS.ESTIMATED_RECORDS)) - .withEstimatedBytes(r.get(STREAM_STATS.ESTIMATED_BYTES)) - .withBytesCommitted(r.get(STREAM_STATS.BYTES_COMMITTED)) - .withRecordsCommitted(r.get(STREAM_STATS.RECORDS_COMMITTED))); + return Map.of(); + } - final var key = new JobAttemptPair(r.get(ATTEMPTS.JOB_ID), r.get(ATTEMPTS.ATTEMPT_NUMBER)); - if (!attemptStats.containsKey(key)) { - LOGGER.error("{} stream stats entry does not have a corresponding sync stats entry. This suggest the database is in a bad state.", key); - return; - } - attemptStats.get(key).perStreamStats().add(streamSyncStats); + final var jobIdsStr = StringUtils.join(jobIds, ','); + return jobDatabase.query(ctx -> { + // Instead of one massive join query, separate this query into two queries for better readability + // for now. + // We can combine the queries at a later date if this still proves to be not efficient enough. + final Map attemptStats = hydrateSyncStats(jobIdsStr, ctx); + hydrateStreamStats(jobIdsStr, ctx, attemptStats); + return attemptStats; }); } + @Override + public SyncStats getAttemptCombinedStats(final long jobId, final int attemptNumber) throws IOException { + return jobDatabase + .query(ctx -> { + final Long attemptId = getAttemptId(jobId, attemptNumber, ctx); + return ctx.select(DSL.asterisk()).from(SYNC_STATS).where(SYNC_STATS.ATTEMPT_ID.eq(attemptId)) + .orderBy(SYNC_STATS.UPDATED_AT.desc()) + .fetchOne(getSyncStatsRecordMapper()); + }); + } + @Override public List getNormalizationSummary(final long jobId, final int attemptNumber) throws IOException { return jobDatabase @@ -669,60 +836,6 @@ public List getNormalizationSummary(final long jobId, fina }); } - @VisibleForTesting - static Long getAttemptId(final long jobId, final int attemptNumber, final DSLContext ctx) { - final Optional record = - ctx.fetch("SELECT id from attempts where job_id = ? AND attempt_number = ?", jobId, - attemptNumber).stream().findFirst(); - if (record.isEmpty()) { - return -1L; - } - - return record.get().get("id", Long.class); - } - - private static RecordMapper getSyncStatsRecordMapper() { - return record -> new SyncStats().withBytesEmitted(record.get(SYNC_STATS.BYTES_EMITTED)).withRecordsEmitted(record.get(SYNC_STATS.RECORDS_EMITTED)) - .withEstimatedBytes(record.get(SYNC_STATS.ESTIMATED_BYTES)).withEstimatedRecords(record.get(SYNC_STATS.ESTIMATED_RECORDS)) - .withSourceStateMessagesEmitted(record.get(SYNC_STATS.SOURCE_STATE_MESSAGES_EMITTED)) - .withDestinationStateMessagesEmitted(record.get(SYNC_STATS.DESTINATION_STATE_MESSAGES_EMITTED)) - .withBytesCommitted(record.get(SYNC_STATS.BYTES_COMMITTED)) - .withRecordsCommitted(record.get(SYNC_STATS.RECORDS_COMMITTED)) - .withMeanSecondsBeforeSourceStateMessageEmitted(record.get(SYNC_STATS.MEAN_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED)) - .withMaxSecondsBeforeSourceStateMessageEmitted(record.get(SYNC_STATS.MAX_SECONDS_BEFORE_SOURCE_STATE_MESSAGE_EMITTED)) - .withMeanSecondsBetweenStateMessageEmittedandCommitted(record.get(SYNC_STATS.MEAN_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED)) - .withMaxSecondsBetweenStateMessageEmittedandCommitted(record.get(SYNC_STATS.MAX_SECONDS_BETWEEN_STATE_MESSAGE_EMITTED_AND_COMMITTED)); - } - - private static RecordMapper getStreamStatsRecordsMapper() { - return record -> { - final var stats = new SyncStats() - .withEstimatedRecords(record.get(STREAM_STATS.ESTIMATED_RECORDS)).withEstimatedBytes(record.get(STREAM_STATS.ESTIMATED_BYTES)) - .withRecordsEmitted(record.get(STREAM_STATS.RECORDS_EMITTED)).withBytesEmitted(record.get(STREAM_STATS.BYTES_EMITTED)) - .withRecordsCommitted(record.get(STREAM_STATS.RECORDS_COMMITTED)).withBytesCommitted(record.get(STREAM_STATS.BYTES_COMMITTED)); - return new StreamSyncStats() - .withStreamName(record.get(STREAM_STATS.STREAM_NAME)).withStreamNamespace(record.get(STREAM_STATS.STREAM_NAMESPACE)) - .withStats(stats); - }; - } - - private static RecordMapper getNormalizationSummaryRecordMapper() { - return record -> { - try { - return new NormalizationSummary().withStartTime(record.get(NORMALIZATION_SUMMARIES.START_TIME).toInstant().toEpochMilli()) - .withEndTime(record.get(NORMALIZATION_SUMMARIES.END_TIME).toInstant().toEpochMilli()) - .withFailures(record.get(NORMALIZATION_SUMMARIES.FAILURES, String.class) == null ? null : deserializeFailureReasons(record)); - } catch (final JsonProcessingException e) { - throw new RuntimeException(e); - } - }; - } - - private static List deserializeFailureReasons(final Record record) throws JsonProcessingException { - final ObjectMapper mapper = new ObjectMapper(); - return List.of(mapper.readValue(String.valueOf(record.get(NORMALIZATION_SUMMARIES.FAILURES)), FailureReason[].class)); - } - @Override public Job getJob(final long jobId) throws IOException { return jobDatabase.query(ctx -> getJob(ctx, jobId)); @@ -828,30 +941,6 @@ public List listJobs(final Set configTypes, }); } - private enum OrderByField { - - CREATED_AT("createdAt"), - UPDATED_AT("updatedAt"); - - private final String field; - - OrderByField(final String field) { - this.field = field; - } - - } - - private enum OrderByMethod { - - ASC, - DESC; - - public static boolean contains(final String method) { - return Arrays.stream(OrderByMethod.values()).anyMatch(enumMethod -> enumMethod.name().equals(method)); - } - - } - @Override public List listJobs(final ConfigType configType, final Instant attemptEndedAtTimestamp) throws IOException { final LocalDateTime timeConvertedIntoLocalDateTime = LocalDateTime.ofInstant(attemptEndedAtTimestamp, ZoneOffset.UTC); @@ -921,6 +1010,22 @@ public List listJobsForConnectionWithStatuses(final UUID connectionId, fina connectionId.toString()))); } + @Override + public List listAttemptsForConnectionAfterTimestamp(final UUID connectionId, + final ConfigType configType, + final Instant attemptEndedAtTimestamp) + throws IOException { + final LocalDateTime timeConvertedIntoLocalDateTime = LocalDateTime.ofInstant(attemptEndedAtTimestamp, ZoneOffset.UTC); + + return jobDatabase.query(ctx -> getAttemptsWithJobsFromResult(ctx.fetch( + BASE_JOB_SELECT_AND_JOIN + WHERE + "CAST(config_type AS VARCHAR) = ? AND " + "scope = ? AND " + "CAST(jobs.status AS VARCHAR) = ? AND " + + " attempts.ended_at > ? " + " ORDER BY attempts.ended_at ASC", + toSqlName(configType), + connectionId.toString(), + toSqlName(JobStatus.SUCCEEDED), + timeConvertedIntoLocalDateTime))); + } + @Override public List listJobStatusAndTimestampWithConnection(final UUID connectionId, final Set configTypes, @@ -1077,125 +1182,6 @@ public List getAttemptNormalizationStatusesForJob(fi Optional.ofNullable(record.get(SYNC_STATS.RECORDS_COMMITTED)), record.get(NORMALIZATION_SUMMARIES.FAILURES) != null))); } - // Retrieves only Job information from the record, without any attempt info - private static Job getJobFromRecord(final Record record) { - return new Job(record.get(JOB_ID, Long.class), - Enums.toEnum(record.get("config_type", String.class), ConfigType.class).orElseThrow(), - record.get("scope", String.class), - parseJobConfigFromString(record.get("config", String.class)), - new ArrayList(), - JobStatus.valueOf(record.get("job_status", String.class).toUpperCase()), - Optional.ofNullable(record.get("job_started_at")).map(value -> getEpoch(record, "started_at")).orElse(null), - getEpoch(record, "job_created_at"), - getEpoch(record, "job_updated_at")); - } - - private static JobConfig parseJobConfigFromString(final String jobConfigString) { - final JobConfig jobConfig = Jsons.deserialize(jobConfigString, JobConfig.class); - // On-the-fly migration of persisted data types related objects (protocol v0->v1) - if (jobConfig.getConfigType() == ConfigType.SYNC && jobConfig.getSync() != null) { - // TODO feature flag this for data types rollout - // CatalogMigrationV1Helper.upgradeSchemaIfNeeded(jobConfig.getSync().getConfiguredAirbyteCatalog()); - CatalogMigrationV1Helper.downgradeSchemaIfNeeded(jobConfig.getSync().getConfiguredAirbyteCatalog()); - } else if (jobConfig.getConfigType() == ConfigType.RESET_CONNECTION && jobConfig.getResetConnection() != null) { - // TODO feature flag this for data types rollout - // CatalogMigrationV1Helper.upgradeSchemaIfNeeded(jobConfig.getResetConnection().getConfiguredAirbyteCatalog()); - CatalogMigrationV1Helper.downgradeSchemaIfNeeded(jobConfig.getResetConnection().getConfiguredAirbyteCatalog()); - } - return jobConfig; - } - - private static Attempt getAttemptFromRecord(final Record record) { - final String attemptOutputString = record.get("attempt_output", String.class); - return new Attempt( - record.get(ATTEMPT_NUMBER, int.class), - record.get(JOB_ID, Long.class), - Path.of(record.get("log_path", String.class)), - record.get("attempt_sync_config", String.class) == null ? null - : Jsons.deserialize(record.get("attempt_sync_config", String.class), AttemptSyncConfig.class), - attemptOutputString == null ? null : parseJobOutputFromString(attemptOutputString), - Enums.toEnum(record.get("attempt_status", String.class), AttemptStatus.class).orElseThrow(), - record.get("processing_task_queue", String.class), - record.get("attempt_failure_summary", String.class) == null ? null - : Jsons.deserialize(record.get("attempt_failure_summary", String.class), AttemptFailureSummary.class), - getEpoch(record, "attempt_created_at"), - getEpoch(record, "attempt_updated_at"), - Optional.ofNullable(record.get("attempt_ended_at")) - .map(value -> getEpoch(record, "attempt_ended_at")) - .orElse(null)); - } - - private static JobOutput parseJobOutputFromString(final String jobOutputString) { - final JobOutput jobOutput = Jsons.deserialize(jobOutputString, JobOutput.class); - // On-the-fly migration of persisted data types related objects (protocol v0->v1) - if (jobOutput.getOutputType() == OutputType.DISCOVER_CATALOG && jobOutput.getDiscoverCatalog() != null) { - // TODO feature flag this for data types rollout - // CatalogMigrationV1Helper.upgradeSchemaIfNeeded(jobOutput.getDiscoverCatalog().getCatalog()); - CatalogMigrationV1Helper.downgradeSchemaIfNeeded(jobOutput.getDiscoverCatalog().getCatalog()); - } else if (jobOutput.getOutputType() == OutputType.SYNC && jobOutput.getSync() != null) { - // TODO feature flag this for data types rollout - // CatalogMigrationV1Helper.upgradeSchemaIfNeeded(jobOutput.getSync().getOutputCatalog()); - CatalogMigrationV1Helper.downgradeSchemaIfNeeded(jobOutput.getSync().getOutputCatalog()); - } - return jobOutput; - } - - private static List getAttemptsWithJobsFromResult(final Result result) { - return result - .stream() - .filter(record -> record.getValue(ATTEMPT_NUMBER) != null) - .map(record -> new AttemptWithJobInfo(getAttemptFromRecord(record), getJobFromRecord(record))) - .collect(Collectors.toList()); - } - - private static List getJobsFromResult(final Result result) { - // keeps results strictly in order so the sql query controls the sort - final List jobs = new ArrayList<>(); - Job currentJob = null; - for (final Record entry : result) { - if (currentJob == null || currentJob.getId() != entry.get(JOB_ID, Long.class)) { - currentJob = getJobFromRecord(entry); - jobs.add(currentJob); - } - if (entry.getValue(ATTEMPT_NUMBER) != null) { - currentJob.getAttempts().add(getAttemptFromRecord(entry)); - } - } - - return jobs; - } - - /** - * Generate a string fragment that can be put in the IN clause of a SQL statement. eg. column IN - * (value1, value2) - * - * @param values to encode - * @param enum type - * @return "'value1', 'value2', 'value3'" - */ - private static > String toSqlInFragment(final Iterable values) { - return StreamSupport.stream(values.spliterator(), false).map(DefaultJobPersistence::toSqlName).map(Names::singleQuote) - .collect(Collectors.joining(",", "(", ")")); - } - - @VisibleForTesting - static > String toSqlName(final T value) { - return value.name().toLowerCase(); - } - - private static > Set configTypeSqlNames(final Set configTypes) { - return configTypes.stream().map(DefaultJobPersistence::toSqlName).collect(Collectors.toSet()); - } - - @VisibleForTesting - static Optional getJobFromResult(final Result result) { - return getJobsFromResult(result).stream().findFirst(); - } - - private static long getEpoch(final Record record, final String fieldName) { - return record.get(fieldName, LocalDateTime.class).toEpochSecond(ZoneOffset.UTC); - } - @Override public Optional getVersion() throws IOException { return getMetadata(AirbyteVersion.AIRBYTE_VERSION_KEY_NAME).findFirst(); @@ -1381,4 +1367,28 @@ private String buildJobOrderByString(final String orderByField, final String ord return String.format("ORDER BY jobs.%s %s ", field, sortMethod); } + private enum OrderByField { + + CREATED_AT("createdAt"), + UPDATED_AT("updatedAt"); + + private final String field; + + OrderByField(final String field) { + this.field = field; + } + + } + + private enum OrderByMethod { + + ASC, + DESC; + + public static boolean contains(final String method) { + return Arrays.stream(OrderByMethod.values()).anyMatch(enumMethod -> enumMethod.name().equals(method)); + } + + } + } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobPersistence.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobPersistence.java index cc04ae97568..8f736eb79a7 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobPersistence.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobPersistence.java @@ -41,22 +41,6 @@ public interface JobPersistence { // SIMPLE GETTERS // - /** - * Convenience POJO for various stats data structures. - * - * @param combinedStats stats for the job - * @param perStreamStats stats for each stream - */ - record AttemptStats(SyncStats combinedStats, List perStreamStats) {} - - /** - * Pair of the job id and attempt number. - * - * @param id job id - * @param attemptNumber attempt number - */ - record JobAttemptPair(long id, int attemptNumber) {} - /** * Retrieve the combined and per stream stats for a single attempt. * @@ -88,10 +72,6 @@ record JobAttemptPair(long id, int attemptNumber) {} Job getJob(long jobId) throws IOException; - // - // JOB LIFECYCLE - // - /** * Enqueue a new job. Its initial status will be pending. * @@ -112,6 +92,10 @@ record JobAttemptPair(long id, int attemptNumber) {} */ void resetJob(long jobId) throws IOException; + // + // JOB LIFECYCLE + // + /** * Set job status from current status to CANCELLED. If already in a terminal status, no op. * @@ -128,10 +112,6 @@ record JobAttemptPair(long id, int attemptNumber) {} */ void failJob(long jobId) throws IOException; - // - // ATTEMPT LIFECYCLE - // - /** * Create a new attempt for a job and return its attempt number. Throws * {@link IllegalStateException} if the job is already in a terminal state. @@ -153,6 +133,10 @@ record JobAttemptPair(long id, int attemptNumber) {} */ void failAttempt(long jobId, int attemptNumber) throws IOException; + // + // ATTEMPT LIFECYCLE + // + /** * Sets an attempt to SUCCEEDED. Also attempts to set the parent job to SUCCEEDED. The job's status * is changed regardless of what state it is in. @@ -163,10 +147,6 @@ record JobAttemptPair(long id, int attemptNumber) {} */ void succeedAttempt(long jobId, int attemptNumber) throws IOException; - // - // END OF LIFECYCLE - // - /** * Sets an attempt's temporal workflow id. Later used to cancel the workflow. */ @@ -177,6 +157,10 @@ record JobAttemptPair(long id, int attemptNumber) {} */ Optional getAttemptTemporalWorkflowId(long jobId, int attemptNumber) throws IOException; + // + // END OF LIFECYCLE + // + /** * Retrieves an Attempt from a given jobId and attemptNumber. */ @@ -316,6 +300,12 @@ List listJobs( List listJobsForConnectionWithStatuses(UUID connectionId, Set configTypes, Set statuses) throws IOException; + // todo: do we actually care about being able to pass in configType + List listAttemptsForConnectionAfterTimestamp(UUID connectionId, + ConfigType configType, + Instant attemptEndedAtTimestamp) + throws IOException; + /** * List job statuses and timestamps for connection id. * @@ -353,7 +343,6 @@ List listJobStatusAndTimestampWithConnection(UUID con * @throws IOException while interacting with the db. */ List listAttemptsWithJobInfo(ConfigType configType, Instant attemptEndedAtTimestamp, final int limit) throws IOException; - /// ARCHIVE /** * Returns the AirbyteVersion. @@ -364,6 +353,7 @@ List listJobStatusAndTimestampWithConnection(UUID con * Set the airbyte version. */ void setVersion(String airbyteVersion) throws IOException; + /// ARCHIVE /** * Get the max supported Airbyte Protocol Version. @@ -394,8 +384,6 @@ List listJobStatusAndTimestampWithConnection(UUID con * Returns a deployment UUID. */ Optional getDeployment() throws IOException; - // a deployment references a setup of airbyte. it is created the first time the docker compose or - // K8s is ready. /** * Set deployment id. If one is already set, the new value is ignored. @@ -406,7 +394,29 @@ List listJobStatusAndTimestampWithConnection(UUID con * Purges job history while ensuring that the latest saved-state information is maintained. */ void purgeJobHistory(); + // a deployment references a setup of airbyte. it is created the first time the docker compose or + // K8s is ready. List getAttemptNormalizationStatusesForJob(final Long jobId) throws IOException; + /** + * Convenience POJO for various stats data structures. + * + * @param combinedStats stats for the job + * @param perStreamStats stats for each stream + */ + record AttemptStats(SyncStats combinedStats, List perStreamStats) { + + } + + /** + * Pair of the job id and attempt number. + * + * @param id job id + * @param attemptNumber attempt number + */ + record JobAttemptPair(long id, int attemptNumber) { + + } + } diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java index 745f88f6244..85766631b50 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java @@ -823,12 +823,7 @@ void testCreateResetConnectionJob() throws IOException { new ConfiguredAirbyteStream() .withStream(CatalogHelpers.createAirbyteStream(STREAM2_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))) .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.OVERWRITE), - // this stream is not being reset, so it should have APPEND destination sync mode - new ConfiguredAirbyteStream() - .withStream(CatalogHelpers.createAirbyteStream(STREAM3_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))) - .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.APPEND))); + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE))); final SyncResourceRequirements expectedSyncResourceRequirements = new SyncResourceRequirements() .withConfigKey(new SyncResourceRequirementsKey().withVariant(DEFAULT_VARIANT)) @@ -890,12 +885,7 @@ void testCreateResetConnectionJobEnsureNoQueuing() throws IOException { new ConfiguredAirbyteStream() .withStream(CatalogHelpers.createAirbyteStream(STREAM2_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))) .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.OVERWRITE), - // this stream is not being reset, so it should have APPEND destination sync mode - new ConfiguredAirbyteStream() - .withStream(CatalogHelpers.createAirbyteStream(STREAM3_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))) - .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.APPEND))); + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE))); final SyncResourceRequirements expectedSyncResourceRequirements = new SyncResourceRequirements() .withConfigKey(new SyncResourceRequirementsKey().withVariant(DEFAULT_VARIANT)) diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java index af1efb5d818..0c3e0396024 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java @@ -193,6 +193,14 @@ private static Job createJob( time); } + private static Supplier incrementingSecondSupplier(final Instant startTime) { + // needs to be an array to work with lambda + final int[] intArray = {0}; + + final Supplier timeSupplier = () -> startTime.plusSeconds(intArray[0]++); + return timeSupplier; + } + @SuppressWarnings("unchecked") @BeforeEach void setup() throws Exception { @@ -384,6 +392,219 @@ void testWriteAttemptFailureSummaryWithUnsupportedUnicode() throws IOException { }); } + @Test + @DisplayName("When getting the last replication job should return the most recently created job") + void testGetLastSyncJobWithMultipleAttempts() throws IOException { + final long jobId = jobPersistence.enqueueJob(SCOPE, SYNC_JOB_CONFIG).orElseThrow(); + jobPersistence.failAttempt(jobId, jobPersistence.createAttempt(jobId, LOG_PATH)); + jobPersistence.failAttempt(jobId, jobPersistence.createAttempt(jobId, LOG_PATH)); + + final Optional actual = jobPersistence.getLastReplicationJob(UUID.fromString(SCOPE)); + + final Job expected = createJob( + jobId, + SYNC_JOB_CONFIG, + JobStatus.INCOMPLETE, + Lists.newArrayList( + createAttempt(0, jobId, AttemptStatus.FAILED, LOG_PATH), + createAttempt(1, jobId, AttemptStatus.FAILED, LOG_PATH)), + NOW.getEpochSecond()); + + assertEquals(Optional.of(expected), actual); + } + + @Test + @DisplayName("Should extract a Job model from a JOOQ result set") + void testGetJobFromRecord() throws IOException, SQLException { + final long jobId = jobPersistence.enqueueJob(SCOPE, SPEC_JOB_CONFIG).orElseThrow(); + + final Optional actual = DefaultJobPersistence.getJobFromResult(getJobRecord(jobId)); + + final Job expected = createJob(jobId, SPEC_JOB_CONFIG, JobStatus.PENDING, Collections.emptyList(), NOW.getEpochSecond()); + assertEquals(Optional.of(expected), actual); + } + + @Test + @DisplayName("Should return correct set of jobs when querying on end timestamp") + void testListJobsWithTimestamp() throws IOException { + // TODO : Once we fix the problem of precision loss in DefaultJobPersistence, change the test value + // to contain milliseconds as well + final Instant now = Instant.parse("2021-01-01T00:00:00Z"); + final Supplier timeSupplier = incrementingSecondSupplier(now); + + jobPersistence = new DefaultJobPersistence(jobDatabase, timeSupplier, DEFAULT_MINIMUM_AGE_IN_DAYS, DEFAULT_EXCESSIVE_NUMBER_OF_JOBS, + DEFAULT_MINIMUM_RECENCY_COUNT); + final long syncJobId = jobPersistence.enqueueJob(SCOPE, SYNC_JOB_CONFIG).orElseThrow(); + final int syncJobAttemptNumber0 = jobPersistence.createAttempt(syncJobId, LOG_PATH); + jobPersistence.failAttempt(syncJobId, syncJobAttemptNumber0); + final Path syncJobSecondAttemptLogPath = LOG_PATH.resolve("2"); + final int syncJobAttemptNumber1 = jobPersistence.createAttempt(syncJobId, syncJobSecondAttemptLogPath); + jobPersistence.failAttempt(syncJobId, syncJobAttemptNumber1); + + final long specJobId = jobPersistence.enqueueJob(SCOPE, SPEC_JOB_CONFIG).orElseThrow(); + final int specJobAttemptNumber0 = jobPersistence.createAttempt(specJobId, LOG_PATH); + jobPersistence.failAttempt(specJobId, specJobAttemptNumber0); + final Path specJobSecondAttemptLogPath = LOG_PATH.resolve("2"); + final int specJobAttemptNumber1 = jobPersistence.createAttempt(specJobId, specJobSecondAttemptLogPath); + jobPersistence.succeedAttempt(specJobId, specJobAttemptNumber1); + + final List jobs = jobPersistence.listJobs(ConfigType.SYNC, Instant.EPOCH); + assertEquals(jobs.size(), 1); + assertEquals(jobs.get(0).getId(), syncJobId); + assertEquals(jobs.get(0).getAttempts().size(), 2); + assertEquals(jobs.get(0).getAttempts().get(0).getAttemptNumber(), 0); + assertEquals(jobs.get(0).getAttempts().get(1).getAttemptNumber(), 1); + + final Path syncJobThirdAttemptLogPath = LOG_PATH.resolve("3"); + final int syncJobAttemptNumber2 = jobPersistence.createAttempt(syncJobId, syncJobThirdAttemptLogPath); + jobPersistence.succeedAttempt(syncJobId, syncJobAttemptNumber2); + + final long newSyncJobId = jobPersistence.enqueueJob(SCOPE, SYNC_JOB_CONFIG).orElseThrow(); + final int newSyncJobAttemptNumber0 = jobPersistence.createAttempt(newSyncJobId, LOG_PATH); + jobPersistence.failAttempt(newSyncJobId, newSyncJobAttemptNumber0); + final Path newSyncJobSecondAttemptLogPath = LOG_PATH.resolve("2"); + final int newSyncJobAttemptNumber1 = jobPersistence.createAttempt(newSyncJobId, newSyncJobSecondAttemptLogPath); + jobPersistence.succeedAttempt(newSyncJobId, newSyncJobAttemptNumber1); + + final Long maxEndedAtTimestamp = + jobs.get(0).getAttempts().stream().map(c -> c.getEndedAtInSecond().orElseThrow()).max(Long::compareTo).orElseThrow(); + + final List secondQueryJobs = jobPersistence.listJobs(ConfigType.SYNC, Instant.ofEpochSecond(maxEndedAtTimestamp)); + assertEquals(secondQueryJobs.size(), 2); + assertEquals(secondQueryJobs.get(0).getId(), syncJobId); + assertEquals(secondQueryJobs.get(0).getAttempts().size(), 1); + assertEquals(secondQueryJobs.get(0).getAttempts().get(0).getAttemptNumber(), 2); + + assertEquals(secondQueryJobs.get(1).getId(), newSyncJobId); + assertEquals(secondQueryJobs.get(1).getAttempts().size(), 2); + assertEquals(secondQueryJobs.get(1).getAttempts().get(0).getAttemptNumber(), 0); + assertEquals(secondQueryJobs.get(1).getAttempts().get(1).getAttemptNumber(), 1); + + Long maxEndedAtTimestampAfterSecondQuery = -1L; + for (final Job c : secondQueryJobs) { + final List attempts = c.getAttempts(); + final Long maxEndedAtTimestampForJob = attempts.stream().map(attempt -> attempt.getEndedAtInSecond().orElseThrow()) + .max(Long::compareTo).orElseThrow(); + if (maxEndedAtTimestampForJob > maxEndedAtTimestampAfterSecondQuery) { + maxEndedAtTimestampAfterSecondQuery = maxEndedAtTimestampForJob; + } + } + + assertEquals(0, jobPersistence.listJobs(ConfigType.SYNC, Instant.ofEpochSecond(maxEndedAtTimestampAfterSecondQuery)).size()); + } + + @Test + @DisplayName("Should return correct list of AttemptWithJobInfo when querying on end timestamp, sorted by attempt end time") + void testListAttemptsWithJobInfo() throws IOException { + final Instant now = Instant.parse("2021-01-01T00:00:00Z"); + final Supplier timeSupplier = incrementingSecondSupplier(now); + jobPersistence = new DefaultJobPersistence(jobDatabase, timeSupplier, DEFAULT_MINIMUM_AGE_IN_DAYS, DEFAULT_EXCESSIVE_NUMBER_OF_JOBS, + DEFAULT_MINIMUM_RECENCY_COUNT); + + final long job1 = jobPersistence.enqueueJob(SCOPE + "-1", SYNC_JOB_CONFIG).orElseThrow(); + final long job2 = jobPersistence.enqueueJob(SCOPE + "-2", SYNC_JOB_CONFIG).orElseThrow(); + + final int job1Attempt1 = jobPersistence.createAttempt(job1, LOG_PATH.resolve("1")); + final int job2Attempt1 = jobPersistence.createAttempt(job2, LOG_PATH.resolve("2")); + jobPersistence.failAttempt(job1, job1Attempt1); + jobPersistence.failAttempt(job2, job2Attempt1); + + final int job1Attempt2 = jobPersistence.createAttempt(job1, LOG_PATH.resolve("3")); + final int job2Attempt2 = jobPersistence.createAttempt(job2, LOG_PATH.resolve("4")); + jobPersistence.failAttempt(job2, job2Attempt2); // job 2 attempt 2 fails before job 1 attempt 2 fails + jobPersistence.failAttempt(job1, job1Attempt2); + + final int job1Attempt3 = jobPersistence.createAttempt(job1, LOG_PATH.resolve("5")); + final int job2Attempt3 = jobPersistence.createAttempt(job2, LOG_PATH.resolve("6")); + jobPersistence.succeedAttempt(job1, job1Attempt3); + jobPersistence.succeedAttempt(job2, job2Attempt3); + + final List allAttempts = jobPersistence.listAttemptsWithJobInfo(ConfigType.SYNC, Instant.ofEpochSecond(0), 1000); + assertEquals(6, allAttempts.size()); + + assertEquals(job1, allAttempts.get(0).getJobInfo().getId()); + assertEquals(job1Attempt1, allAttempts.get(0).getAttempt().getAttemptNumber()); + + assertEquals(job2, allAttempts.get(1).getJobInfo().getId()); + assertEquals(job2Attempt1, allAttempts.get(1).getAttempt().getAttemptNumber()); + + assertEquals(job2, allAttempts.get(2).getJobInfo().getId()); + assertEquals(job2Attempt2, allAttempts.get(2).getAttempt().getAttemptNumber()); + + assertEquals(job1, allAttempts.get(3).getJobInfo().getId()); + assertEquals(job1Attempt2, allAttempts.get(3).getAttempt().getAttemptNumber()); + + assertEquals(job1, allAttempts.get(4).getJobInfo().getId()); + assertEquals(job1Attempt3, allAttempts.get(4).getAttempt().getAttemptNumber()); + + assertEquals(job2, allAttempts.get(5).getJobInfo().getId()); + assertEquals(job2Attempt3, allAttempts.get(5).getAttempt().getAttemptNumber()); + + final List attemptsAfterTimestamp = jobPersistence.listAttemptsWithJobInfo(ConfigType.SYNC, + Instant.ofEpochSecond(allAttempts.get(2).getAttempt().getEndedAtInSecond().orElseThrow()), 1000); + assertEquals(3, attemptsAfterTimestamp.size()); + + assertEquals(job1, attemptsAfterTimestamp.get(0).getJobInfo().getId()); + assertEquals(job1Attempt2, attemptsAfterTimestamp.get(0).getAttempt().getAttemptNumber()); + + assertEquals(job1, attemptsAfterTimestamp.get(1).getJobInfo().getId()); + assertEquals(job1Attempt3, attemptsAfterTimestamp.get(1).getAttempt().getAttemptNumber()); + + assertEquals(job2, attemptsAfterTimestamp.get(2).getJobInfo().getId()); + assertEquals(job2Attempt3, attemptsAfterTimestamp.get(2).getAttempt().getAttemptNumber()); + } + + @Test + void testAirbyteProtocolVersionMaxMetadata() throws IOException { + assertTrue(jobPersistence.getAirbyteProtocolVersionMax().isEmpty()); + + final Version maxVersion1 = new Version("0.1.0"); + jobPersistence.setAirbyteProtocolVersionMax(maxVersion1); + final Optional maxVersion1read = jobPersistence.getAirbyteProtocolVersionMax(); + assertEquals(maxVersion1, maxVersion1read.orElseThrow()); + + final Version maxVersion2 = new Version("1.2.1"); + jobPersistence.setAirbyteProtocolVersionMax(maxVersion2); + final Optional maxVersion2read = jobPersistence.getAirbyteProtocolVersionMax(); + assertEquals(maxVersion2, maxVersion2read.orElseThrow()); + } + + @Test + void testAirbyteProtocolVersionMinMetadata() throws IOException { + assertTrue(jobPersistence.getAirbyteProtocolVersionMin().isEmpty()); + + final Version minVersion1 = new Version("1.1.0"); + jobPersistence.setAirbyteProtocolVersionMin(minVersion1); + final Optional minVersion1read = jobPersistence.getAirbyteProtocolVersionMin(); + assertEquals(minVersion1, minVersion1read.orElseThrow()); + + final Version minVersion2 = new Version("3.0.1"); + jobPersistence.setAirbyteProtocolVersionMin(minVersion2); + final Optional minVersion2read = jobPersistence.getAirbyteProtocolVersionMin(); + assertEquals(minVersion2, minVersion2read.orElseThrow()); + } + + @Test + void testAirbyteProtocolVersionRange() throws IOException { + final Version v1 = new Version("1.5.0"); + final Version v2 = new Version("2.5.0"); + final Optional range = jobPersistence.getCurrentProtocolVersionRange(); + assertEquals(Optional.empty(), range); + + jobPersistence.setAirbyteProtocolVersionMax(v2); + final Optional range2 = jobPersistence.getCurrentProtocolVersionRange(); + assertEquals(Optional.of(new AirbyteProtocolVersionRange(AirbyteProtocolVersion.DEFAULT_AIRBYTE_PROTOCOL_VERSION, v2)), range2); + + jobPersistence.setAirbyteProtocolVersionMin(v1); + final Optional range3 = jobPersistence.getCurrentProtocolVersionRange(); + assertEquals(Optional.of(new AirbyteProtocolVersionRange(v1, v2)), range3); + } + + private long createJobAt(final Instant createdAt) throws IOException { + when(timeSupplier.get()).thenReturn(createdAt); + return jobPersistence.enqueueJob(SCOPE, SPEC_JOB_CONFIG).orElseThrow(); + } + @Nested @DisplayName("Stats Related Tests") class Stats { @@ -417,7 +638,7 @@ void testWriteStatsFirst() throws IOException { assertEquals(bytesCommitted, combined.getBytesCommitted()); // As of this writing, committed and state messages are not expected. - assertEquals(null, combined.getDestinationStateMessagesEmitted()); + assertNull(combined.getDestinationStateMessagesEmitted()); final var actStreamStats = stats.perStreamStats(); assertEquals(2, actStreamStats.size()); @@ -707,227 +928,6 @@ void testGetAttemptCombinedStats() throws IOException { } - @Test - @DisplayName("When getting the last replication job should return the most recently created job") - void testGetLastSyncJobWithMultipleAttempts() throws IOException { - final long jobId = jobPersistence.enqueueJob(SCOPE, SYNC_JOB_CONFIG).orElseThrow(); - jobPersistence.failAttempt(jobId, jobPersistence.createAttempt(jobId, LOG_PATH)); - jobPersistence.failAttempt(jobId, jobPersistence.createAttempt(jobId, LOG_PATH)); - - final Optional actual = jobPersistence.getLastReplicationJob(UUID.fromString(SCOPE)); - - final Job expected = createJob( - jobId, - SYNC_JOB_CONFIG, - JobStatus.INCOMPLETE, - Lists.newArrayList( - createAttempt(0, jobId, AttemptStatus.FAILED, LOG_PATH), - createAttempt(1, jobId, AttemptStatus.FAILED, LOG_PATH)), - NOW.getEpochSecond()); - - assertEquals(Optional.of(expected), actual); - } - - @Test - @DisplayName("Should extract a Job model from a JOOQ result set") - void testGetJobFromRecord() throws IOException, SQLException { - final long jobId = jobPersistence.enqueueJob(SCOPE, SPEC_JOB_CONFIG).orElseThrow(); - - final Optional actual = DefaultJobPersistence.getJobFromResult(getJobRecord(jobId)); - - final Job expected = createJob(jobId, SPEC_JOB_CONFIG, JobStatus.PENDING, Collections.emptyList(), NOW.getEpochSecond()); - assertEquals(Optional.of(expected), actual); - } - - @Test - @DisplayName("Should return correct set of jobs when querying on end timestamp") - void testListJobsWithTimestamp() throws IOException { - // TODO : Once we fix the problem of precision loss in DefaultJobPersistence, change the test value - // to contain milliseconds as well - final Instant now = Instant.parse("2021-01-01T00:00:00Z"); - final Supplier timeSupplier = incrementingSecondSupplier(now); - - jobPersistence = new DefaultJobPersistence(jobDatabase, timeSupplier, DEFAULT_MINIMUM_AGE_IN_DAYS, DEFAULT_EXCESSIVE_NUMBER_OF_JOBS, - DEFAULT_MINIMUM_RECENCY_COUNT); - final long syncJobId = jobPersistence.enqueueJob(SCOPE, SYNC_JOB_CONFIG).orElseThrow(); - final int syncJobAttemptNumber0 = jobPersistence.createAttempt(syncJobId, LOG_PATH); - jobPersistence.failAttempt(syncJobId, syncJobAttemptNumber0); - final Path syncJobSecondAttemptLogPath = LOG_PATH.resolve("2"); - final int syncJobAttemptNumber1 = jobPersistence.createAttempt(syncJobId, syncJobSecondAttemptLogPath); - jobPersistence.failAttempt(syncJobId, syncJobAttemptNumber1); - - final long specJobId = jobPersistence.enqueueJob(SCOPE, SPEC_JOB_CONFIG).orElseThrow(); - final int specJobAttemptNumber0 = jobPersistence.createAttempt(specJobId, LOG_PATH); - jobPersistence.failAttempt(specJobId, specJobAttemptNumber0); - final Path specJobSecondAttemptLogPath = LOG_PATH.resolve("2"); - final int specJobAttemptNumber1 = jobPersistence.createAttempt(specJobId, specJobSecondAttemptLogPath); - jobPersistence.succeedAttempt(specJobId, specJobAttemptNumber1); - - final List jobs = jobPersistence.listJobs(ConfigType.SYNC, Instant.EPOCH); - assertEquals(jobs.size(), 1); - assertEquals(jobs.get(0).getId(), syncJobId); - assertEquals(jobs.get(0).getAttempts().size(), 2); - assertEquals(jobs.get(0).getAttempts().get(0).getAttemptNumber(), 0); - assertEquals(jobs.get(0).getAttempts().get(1).getAttemptNumber(), 1); - - final Path syncJobThirdAttemptLogPath = LOG_PATH.resolve("3"); - final int syncJobAttemptNumber2 = jobPersistence.createAttempt(syncJobId, syncJobThirdAttemptLogPath); - jobPersistence.succeedAttempt(syncJobId, syncJobAttemptNumber2); - - final long newSyncJobId = jobPersistence.enqueueJob(SCOPE, SYNC_JOB_CONFIG).orElseThrow(); - final int newSyncJobAttemptNumber0 = jobPersistence.createAttempt(newSyncJobId, LOG_PATH); - jobPersistence.failAttempt(newSyncJobId, newSyncJobAttemptNumber0); - final Path newSyncJobSecondAttemptLogPath = LOG_PATH.resolve("2"); - final int newSyncJobAttemptNumber1 = jobPersistence.createAttempt(newSyncJobId, newSyncJobSecondAttemptLogPath); - jobPersistence.succeedAttempt(newSyncJobId, newSyncJobAttemptNumber1); - - final Long maxEndedAtTimestamp = - jobs.get(0).getAttempts().stream().map(c -> c.getEndedAtInSecond().orElseThrow()).max(Long::compareTo).orElseThrow(); - - final List secondQueryJobs = jobPersistence.listJobs(ConfigType.SYNC, Instant.ofEpochSecond(maxEndedAtTimestamp)); - assertEquals(secondQueryJobs.size(), 2); - assertEquals(secondQueryJobs.get(0).getId(), syncJobId); - assertEquals(secondQueryJobs.get(0).getAttempts().size(), 1); - assertEquals(secondQueryJobs.get(0).getAttempts().get(0).getAttemptNumber(), 2); - - assertEquals(secondQueryJobs.get(1).getId(), newSyncJobId); - assertEquals(secondQueryJobs.get(1).getAttempts().size(), 2); - assertEquals(secondQueryJobs.get(1).getAttempts().get(0).getAttemptNumber(), 0); - assertEquals(secondQueryJobs.get(1).getAttempts().get(1).getAttemptNumber(), 1); - - Long maxEndedAtTimestampAfterSecondQuery = -1L; - for (final Job c : secondQueryJobs) { - final List attempts = c.getAttempts(); - final Long maxEndedAtTimestampForJob = attempts.stream().map(attempt -> attempt.getEndedAtInSecond().orElseThrow()) - .max(Long::compareTo).orElseThrow(); - if (maxEndedAtTimestampForJob > maxEndedAtTimestampAfterSecondQuery) { - maxEndedAtTimestampAfterSecondQuery = maxEndedAtTimestampForJob; - } - } - - assertEquals(0, jobPersistence.listJobs(ConfigType.SYNC, Instant.ofEpochSecond(maxEndedAtTimestampAfterSecondQuery)).size()); - } - - @Test - @DisplayName("Should return correct list of AttemptWithJobInfo when querying on end timestamp, sorted by attempt end time") - void testListAttemptsWithJobInfo() throws IOException { - final Instant now = Instant.parse("2021-01-01T00:00:00Z"); - final Supplier timeSupplier = incrementingSecondSupplier(now); - jobPersistence = new DefaultJobPersistence(jobDatabase, timeSupplier, DEFAULT_MINIMUM_AGE_IN_DAYS, DEFAULT_EXCESSIVE_NUMBER_OF_JOBS, - DEFAULT_MINIMUM_RECENCY_COUNT); - - final long job1 = jobPersistence.enqueueJob(SCOPE + "-1", SYNC_JOB_CONFIG).orElseThrow(); - final long job2 = jobPersistence.enqueueJob(SCOPE + "-2", SYNC_JOB_CONFIG).orElseThrow(); - - final int job1Attempt1 = jobPersistence.createAttempt(job1, LOG_PATH.resolve("1")); - final int job2Attempt1 = jobPersistence.createAttempt(job2, LOG_PATH.resolve("2")); - jobPersistence.failAttempt(job1, job1Attempt1); - jobPersistence.failAttempt(job2, job2Attempt1); - - final int job1Attempt2 = jobPersistence.createAttempt(job1, LOG_PATH.resolve("3")); - final int job2Attempt2 = jobPersistence.createAttempt(job2, LOG_PATH.resolve("4")); - jobPersistence.failAttempt(job2, job2Attempt2); // job 2 attempt 2 fails before job 1 attempt 2 fails - jobPersistence.failAttempt(job1, job1Attempt2); - - final int job1Attempt3 = jobPersistence.createAttempt(job1, LOG_PATH.resolve("5")); - final int job2Attempt3 = jobPersistence.createAttempt(job2, LOG_PATH.resolve("6")); - jobPersistence.succeedAttempt(job1, job1Attempt3); - jobPersistence.succeedAttempt(job2, job2Attempt3); - - final List allAttempts = jobPersistence.listAttemptsWithJobInfo(ConfigType.SYNC, Instant.ofEpochSecond(0), 1000); - assertEquals(6, allAttempts.size()); - - assertEquals(job1, allAttempts.get(0).getJobInfo().getId()); - assertEquals(job1Attempt1, allAttempts.get(0).getAttempt().getAttemptNumber()); - - assertEquals(job2, allAttempts.get(1).getJobInfo().getId()); - assertEquals(job2Attempt1, allAttempts.get(1).getAttempt().getAttemptNumber()); - - assertEquals(job2, allAttempts.get(2).getJobInfo().getId()); - assertEquals(job2Attempt2, allAttempts.get(2).getAttempt().getAttemptNumber()); - - assertEquals(job1, allAttempts.get(3).getJobInfo().getId()); - assertEquals(job1Attempt2, allAttempts.get(3).getAttempt().getAttemptNumber()); - - assertEquals(job1, allAttempts.get(4).getJobInfo().getId()); - assertEquals(job1Attempt3, allAttempts.get(4).getAttempt().getAttemptNumber()); - - assertEquals(job2, allAttempts.get(5).getJobInfo().getId()); - assertEquals(job2Attempt3, allAttempts.get(5).getAttempt().getAttemptNumber()); - - final List attemptsAfterTimestamp = jobPersistence.listAttemptsWithJobInfo(ConfigType.SYNC, - Instant.ofEpochSecond(allAttempts.get(2).getAttempt().getEndedAtInSecond().orElseThrow()), 1000); - assertEquals(3, attemptsAfterTimestamp.size()); - - assertEquals(job1, attemptsAfterTimestamp.get(0).getJobInfo().getId()); - assertEquals(job1Attempt2, attemptsAfterTimestamp.get(0).getAttempt().getAttemptNumber()); - - assertEquals(job1, attemptsAfterTimestamp.get(1).getJobInfo().getId()); - assertEquals(job1Attempt3, attemptsAfterTimestamp.get(1).getAttempt().getAttemptNumber()); - - assertEquals(job2, attemptsAfterTimestamp.get(2).getJobInfo().getId()); - assertEquals(job2Attempt3, attemptsAfterTimestamp.get(2).getAttempt().getAttemptNumber()); - } - - private static Supplier incrementingSecondSupplier(final Instant startTime) { - // needs to be an array to work with lambda - final int[] intArray = {0}; - - final Supplier timeSupplier = () -> startTime.plusSeconds(intArray[0]++); - return timeSupplier; - } - - @Test - void testAirbyteProtocolVersionMaxMetadata() throws IOException { - assertTrue(jobPersistence.getAirbyteProtocolVersionMax().isEmpty()); - - final Version maxVersion1 = new Version("0.1.0"); - jobPersistence.setAirbyteProtocolVersionMax(maxVersion1); - final Optional maxVersion1read = jobPersistence.getAirbyteProtocolVersionMax(); - assertEquals(maxVersion1, maxVersion1read.orElseThrow()); - - final Version maxVersion2 = new Version("1.2.1"); - jobPersistence.setAirbyteProtocolVersionMax(maxVersion2); - final Optional maxVersion2read = jobPersistence.getAirbyteProtocolVersionMax(); - assertEquals(maxVersion2, maxVersion2read.orElseThrow()); - } - - @Test - void testAirbyteProtocolVersionMinMetadata() throws IOException { - assertTrue(jobPersistence.getAirbyteProtocolVersionMin().isEmpty()); - - final Version minVersion1 = new Version("1.1.0"); - jobPersistence.setAirbyteProtocolVersionMin(minVersion1); - final Optional minVersion1read = jobPersistence.getAirbyteProtocolVersionMin(); - assertEquals(minVersion1, minVersion1read.orElseThrow()); - - final Version minVersion2 = new Version("3.0.1"); - jobPersistence.setAirbyteProtocolVersionMin(minVersion2); - final Optional minVersion2read = jobPersistence.getAirbyteProtocolVersionMin(); - assertEquals(minVersion2, minVersion2read.orElseThrow()); - } - - @Test - void testAirbyteProtocolVersionRange() throws IOException { - final Version v1 = new Version("1.5.0"); - final Version v2 = new Version("2.5.0"); - final Optional range = jobPersistence.getCurrentProtocolVersionRange(); - assertEquals(Optional.empty(), range); - - jobPersistence.setAirbyteProtocolVersionMax(v2); - final Optional range2 = jobPersistence.getCurrentProtocolVersionRange(); - assertEquals(Optional.of(new AirbyteProtocolVersionRange(AirbyteProtocolVersion.DEFAULT_AIRBYTE_PROTOCOL_VERSION, v2)), range2); - - jobPersistence.setAirbyteProtocolVersionMin(v1); - final Optional range3 = jobPersistence.getCurrentProtocolVersionRange(); - assertEquals(Optional.of(new AirbyteProtocolVersionRange(v1, v2)), range3); - } - - private long createJobAt(final Instant createdAt) throws IOException { - when(timeSupplier.get()).thenReturn(createdAt); - return jobPersistence.enqueueJob(SCOPE, SPEC_JOB_CONFIG).orElseThrow(); - } - @Nested class TemporalWorkflowInfo { @@ -1173,6 +1173,40 @@ void testGetAttemptMultiple() throws IOException { } + @Nested + @DisplayName("List attempts after a given timestamp for a given connection") + class ListAttemptsByConnectionByTimestamp { + + @Test + @DisplayName("Returns only entries after the timestamp") + void testListAttemptsForConnectionAfterTimestamp() throws IOException { + + final long jobId1 = jobPersistence.enqueueJob(SCOPE, SYNC_JOB_CONFIG).orElseThrow(); + final int attemptId1 = jobPersistence.createAttempt(jobId1, LOG_PATH); + jobPersistence.succeedAttempt(jobId1, attemptId1); + + final Instant addTwoSeconds = NOW.plusSeconds(2); + when(timeSupplier.get()).thenReturn(addTwoSeconds); + final Instant afterNow = NOW; + + final long jobId2 = jobPersistence.enqueueJob(SCOPE, SYNC_JOB_CONFIG).orElseThrow(); + final int attemptId2 = jobPersistence.createAttempt(jobId2, LOG_PATH); + jobPersistence.succeedAttempt(jobId2, attemptId2); + + final long jobId3 = jobPersistence.enqueueJob(SCOPE, SYNC_JOB_CONFIG).orElseThrow(); + final int attemptId3 = jobPersistence.createAttempt(jobId3, LOG_PATH); + jobPersistence.succeedAttempt(jobId3, attemptId3); + + final List attempts = jobPersistence.listAttemptsForConnectionAfterTimestamp(CONNECTION_ID, ConfigType.SYNC, + afterNow); + + assertEquals(2, attempts.size()); + assertEquals(jobId2, attempts.get(0).getJobInfo().getId()); + assertEquals(jobId3, attempts.get(1).getJobInfo().getId()); + } + + } + @Nested @DisplayName("When enqueueing job") class EnqueueJob { @@ -2174,11 +2208,11 @@ void testPurgeJobHistory(final int numJobs, addStateToJob(allJobs.get(lastStatePosition + 1)); // sanity check that the attempt does have saved state so the purge history sql detects it correctly - assertTrue(lastJobWithState.getAttempts().get(0).getOutput() != null, + assertNotNull(lastJobWithState.getAttempts().get(0).getOutput(), goalOfTestScenario + " - missing saved state on job that was supposed to have it."); // Execute the job history purge and check what jobs are left. - ((DefaultJobPersistence) jobPersistence).purgeJobHistory(fakeNow); + jobPersistence.purgeJobHistory(fakeNow); final List afterPurge = jobPersistence.listJobs(Set.of(ConfigType.SYNC), currentScope, 9999); // Test - contains expected number of jobs and no more than that diff --git a/airbyte-proxy/build.gradle b/airbyte-proxy/build.gradle deleted file mode 100644 index 50f0b938d42..00000000000 --- a/airbyte-proxy/build.gradle +++ /dev/null @@ -1,38 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -airbyte { - docker { - imageName = "proxy" - } -} - -def prepareBuild = tasks.register("prepareBuild", Copy) { - from layout.projectDirectory.file("nginx-auth.conf.template") - from layout.projectDirectory.file("nginx-no-auth.conf.template") - from layout.projectDirectory.file("run.sh") - from layout.projectDirectory.file("401.html") - - into layout.buildDirectory.dir("airbyte/docker") -} - -tasks.named("dockerBuildImage") { - dependsOn prepareBuild - inputs.file "../.env" -} - -def bashTest = tasks.register("bashTest", Exec) { - inputs.file(layout.projectDirectory.file("nginx-auth.conf.template")) - inputs.file(layout.projectDirectory.file("nginx-no-auth.conf.template")) - inputs.file(layout.projectDirectory.file("run.sh")) - inputs.file(layout.projectDirectory.file("401.html")) - outputs.upToDateWhen { true } - dependsOn tasks.named("dockerBuildImage") - commandLine "./test.sh" -} - -// we can't override the 'test' command, so we can make our bash test a dependency -test.dependsOn(bashTest) diff --git a/airbyte-proxy/build.gradle.kts b/airbyte-proxy/build.gradle.kts new file mode 100644 index 00000000000..95fd2f83bad --- /dev/null +++ b/airbyte-proxy/build.gradle.kts @@ -0,0 +1,40 @@ +plugins { + id("io.airbyte.gradle.jvm") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +airbyte { + docker { + imageName = "proxy" + } +} + +val prepareBuild = tasks.register("prepareBuild") { + from(layout.projectDirectory.file("nginx-auth.conf.template")) + from(layout.projectDirectory.file("nginx-no-auth.conf.template")) + from(layout.projectDirectory.file("run.sh")) + from(layout.projectDirectory.file("401.html")) + + into(layout.buildDirectory.dir("airbyte/docker")) +} + +tasks.named("dockerBuildImage") { + dependsOn(prepareBuild) + inputs.file("../.env") +} + +val bashTest = tasks.register("bashTest") { + inputs.file(layout.projectDirectory.file("nginx-auth.conf.template")) + inputs.file(layout.projectDirectory.file("nginx-no-auth.conf.template")) + inputs.file(layout.projectDirectory.file("run.sh")) + inputs.file(layout.projectDirectory.file("401.html")) + outputs.upToDateWhen { true } + dependsOn(tasks.named("dockerBuildImage")) + commandLine("./test.sh") +} + +// we can"t override the "test" command, so we can make our bash test a dependency) +tasks.named("test") { + dependsOn(bashTest) +} diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle deleted file mode 100644 index b0c6e835ccf..00000000000 --- a/airbyte-server/build.gradle +++ /dev/null @@ -1,155 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.app" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -configurations.all { - resolutionStrategy { - // Ensure that the versions defined in deps.toml are used - // instead of versions from transitive dependencies - // Force to avoid updated version brought in transitively from Micronaut 3.8+ - // that is incompatible with our current Helm setup - force libs.flyway.core, libs.s3, libs.aws.java.sdk.s3, libs.sts, libs.aws.java.sdk.sts - } -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.lombok - annotationProcessor libs.bundles.micronaut.annotation.processor - annotationProcessor libs.micronaut.jaxrs.processor - - implementation libs.reactor.core - compileOnly libs.lombok - - testCompileOnly libs.lombok - testAnnotationProcessor libs.lombok - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - implementation libs.bundles.micronaut.data.jdbc - implementation libs.micronaut.jaxrs.server - implementation libs.micronaut.security - implementation libs.flyway.core - implementation libs.s3 - implementation libs.sts - implementation libs.aws.java.sdk.s3 - implementation libs.aws.java.sdk.sts - - implementation project(':airbyte-analytics') - implementation project(':airbyte-api') - implementation project(':airbyte-commons') - implementation project(':airbyte-commons-auth') - implementation project(':airbyte-commons-converters') - implementation project(':airbyte-commons-license') - implementation project(':airbyte-commons-micronaut') - implementation project(':airbyte-commons-temporal') - implementation project(':airbyte-commons-temporal-core') - implementation project(':airbyte-commons-server') - implementation project(':airbyte-commons-with-dependencies') - implementation project(':airbyte-config:init') - implementation project(':airbyte-config:config-models') - implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-config:config-secrets') - implementation project(':airbyte-config:specs') - implementation project(':airbyte-data') - implementation project(":airbyte-featureflag") - implementation project(':airbyte-metrics:metrics-lib') - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-db:jooq') - implementation project(":airbyte-json-validation") - implementation project(':airbyte-notification') - implementation project(':airbyte-oauth') - implementation libs.airbyte.protocol - implementation project(':airbyte-persistence:job-persistence') - - implementation libs.slugify - implementation libs.temporal.sdk - implementation libs.bundles.datadog - implementation libs.sentry.java - implementation libs.swagger.annotations - implementation libs.google.cloud.storage - - runtimeOnly libs.javax.databind - - // Required for local database secret hydration - runtimeOnly(libs.hikaricp) - runtimeOnly(libs.h2.database) - - testImplementation libs.bundles.micronaut.test - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - testImplementation project(':airbyte-test-utils') - testImplementation libs.bundles.micronaut.test - testImplementation libs.postgresql - testImplementation libs.platform.testcontainers.postgresql - testImplementation libs.mockwebserver - testImplementation libs.mockito.inline - - testImplementation libs.reactor.test - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer -} - -// we want to be able to access the generated db files from config/init when we build the server docker image. -def copySeed = tasks.register("copySeed", Copy) { - from "${project(':airbyte-config:init').buildDir}/resources/main/config" - into "${buildDir}/config_init/resources/main/config" - dependsOn(project(':airbyte-config:init').processResources) -} - -// need to make sure that the files are in the resource directory before copying. -// tests require the seed to exist. -test.dependsOn(copySeed) -assemble.dependsOn(copySeed) - - -Properties env = new Properties() -rootProject.file('.env.dev').withInputStream { env.load(it) } - -airbyte { - application { - mainClass = "io.airbyte.server.Application" - defaultJvmArgs = ["-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0"] - localEnvVars = env + [ - "AIRBYTE_ROLE" : System.getenv("AIRBYTE_ROLE") ?: "undefined", - "AIRBYTE_VERSION" : env.VERSION, - "DATABASE_USER" : env.DATABASE_USER, - "DATABASE_PASSWORD" : env.DATABASE_PASSWORD, - "CONFIG_DATABASE_USER" : env.CONFIG_DATABASE_USER ?: "", - "CONFIG_DATABASE_PASSWORD" : env.CONFIG_DATABASE_PASSWORD ?: "", - // we map the docker pg db to port 5433 so it does not conflict with other pg instances. - "DATABASE_URL" : "jdbc:postgresql://localhost:5433/${env.DATABASE_DB}", - "CONFIG_DATABASE_URL" : "jdbc:postgresql://localhost:5433/${env.CONFIG_DATABASE_DB}", - "RUN_DATABASE_MIGRATION_ON_STARTUP": "true", - "WORKSPACE_ROOT" : env.WORKSPACE_ROOT, - "CONFIG_ROOT" : "/tmp/airbyte_config", - "TRACKING_STRATEGY" : env.TRACKING_STRATEGY, - "TEMPORAL_HOST" : "localhost:7233", - "MICRONAUT_ENVIRONMENTS" : "control-plane" - ] as Map - } - - docker { - imageName = "server" - } - - spotbugs { - excludes = [" \n" + - " \n" + - " \n" + - " \n" + - " "] - } -} - -test { - environment 'AIRBYTE_VERSION', env.VERSION - environment 'MICRONAUT_ENVIRONMENTS', 'test' - environment 'SERVICE_NAME', project.name -} diff --git a/airbyte-server/build.gradle.kts b/airbyte-server/build.gradle.kts new file mode 100644 index 00000000000..20eed224889 --- /dev/null +++ b/airbyte-server/build.gradle.kts @@ -0,0 +1,165 @@ +import java.util.Properties + +plugins { + id("io.airbyte.gradle.jvm.app") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +configurations.all { + resolutionStrategy { + // Ensure that the versions defined in deps.toml are used) + // instead of versions from transitive dependencies) + // Force to avoid updated version(brought in transitively from Micronaut 3.8+) + // that is incompatible with our current Helm setup) + force (libs.flyway.core, libs.s3, libs.aws.java.sdk.s3, libs.sts, libs.aws.java.sdk.sts) + } +} + +dependencies { + annotationProcessor(platform(libs.micronaut.bom)) + annotationProcessor(libs.lombok) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + annotationProcessor(libs.micronaut.jaxrs.processor) + + implementation(libs.reactor.core) + compileOnly(libs.lombok) + + testCompileOnly(libs.lombok) + testAnnotationProcessor(libs.lombok) + + implementation(platform(libs.micronaut.bom)) + implementation(libs.bundles.micronaut) + implementation(libs.bundles.micronaut.data.jdbc) + implementation(libs.micronaut.jaxrs.server) + implementation(libs.micronaut.security) + implementation(libs.flyway.core) + implementation(libs.s3) + implementation(libs.sts) + implementation(libs.aws.java.sdk.s3) + implementation(libs.aws.java.sdk.sts) + + implementation(project(":airbyte-analytics")) + implementation(project(":airbyte-api")) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-auth")) + implementation(project(":airbyte-commons-converters")) + implementation(project(":airbyte-commons-license")) + implementation(project(":airbyte-commons-micronaut")) + implementation(project(":airbyte-commons-temporal")) + implementation(project(":airbyte-commons-temporal-core")) + implementation(project(":airbyte-commons-server")) + implementation(project(":airbyte-commons-with-dependencies")) + implementation(project(":airbyte-config:init")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-config:config-secrets")) + implementation(project(":airbyte-config:specs")) + implementation(project(":airbyte-data")) + implementation(project(":airbyte-featureflag")) + implementation(project(":airbyte-metrics:metrics-lib")) + implementation(project(":airbyte-db:db-lib")) + implementation(project(":airbyte-db:jooq")) + implementation(project(":airbyte-json-validation")) + implementation(project(":airbyte-notification")) + implementation(project(":airbyte-oauth")) + implementation(libs.airbyte.protocol) + implementation(project(":airbyte-persistence:job-persistence")) + + implementation(libs.slugify) + implementation(libs.temporal.sdk) + implementation(libs.bundles.datadog) + implementation(libs.sentry.java) + implementation(libs.swagger.annotations) + implementation(libs.google.cloud.storage) + + runtimeOnly(libs.javax.databind) + + // Required for local database secret hydration) + runtimeOnly(libs.hikaricp) + runtimeOnly(libs.h2.database) + + testImplementation(libs.bundles.micronaut.test) + testAnnotationProcessor(platform(libs.micronaut.bom)) + testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) + + testImplementation(project(":airbyte-test-utils")) + testImplementation(libs.bundles.micronaut.test) + testImplementation(libs.postgresql) + testImplementation(libs.platform.testcontainers.postgresql) + testImplementation(libs.mockwebserver) + testImplementation(libs.mockito.inline) + + testImplementation(libs.reactor.test) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) +} + +// we want to be able to access the generated db files from config/init when we build the server docker image.) +val copySeed = tasks.register("copySeed") { + from ("${project(":airbyte-config:init").buildDir}/resources/main/config") + into ("$buildDir/config_init/resources/main/config") + dependsOn(project(":airbyte-config:init").tasks.named("processResources")) +} + +// need to make sure that the files are in the resource directory before copying.) +// tests require the seed to exist.) +tasks.named("test") { + dependsOn(copySeed) +} +tasks.named("assemble") { + dependsOn(copySeed) +} + +val env = Properties().apply { + load(rootProject.file(".env.dev").inputStream()) +} + +airbyte { + application { + mainClass = "io.airbyte.server.Application" + defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + @Suppress("UNCHECKED_CAST") + localEnvVars.putAll(env.toMap() as Map) + localEnvVars.putAll(mapOf( + "AIRBYTE_ROLE" to (System.getenv("AIRBYTE_ROLE") ?: "undefined"), + "AIRBYTE_VERSION" to env["VERSION"].toString(), + "DATABASE_USER" to env["DATABASE_USER"].toString(), + "DATABASE_PASSWORD" to env["DATABASE_PASSWORD"].toString(), + "CONFIG_DATABASE_USER" to (env["CONFIG_DATABASE_USER"]?.toString() ?: ""), + "CONFIG_DATABASE_PASSWORD" to (env["CONFIG_DATABASE_PASSWORD"]?.toString() ?: ""), + // we map the docker pg db to port 5433 so it does not conflict with other pg instances. + "DATABASE_URL" to "jdbc:postgresql://localhost:5433/${env["DATABASE_DB"]}", + "CONFIG_DATABASE_URL" to "jdbc:postgresql://localhost:5433/${env["CONFIG_DATABASE_DB"]}", + "RUN_DATABASE_MIGRATION_ON_STARTUP" to "true", + "WORKSPACE_ROOT" to env["WORKSPACE_ROOT"].toString(), + "CONFIG_ROOT" to "/tmp/airbyte_config", + "TRACKING_STRATEGY" to env["TRACKING_STRATEGY"].toString(), + "TEMPORAL_HOST" to "localhost:7233", + "MICRONAUT_ENVIRONMENTS" to "control-plane", + )) + } + + docker { + imageName = "server" + } + + spotbugs { + excludes = listOf(" \n" + + " \n" + + " \n" + + " \n" + + " ") + } +} + +tasks.named("test") { + environment(mapOf( + "AIRBYTE_VERSION" to env["VERSION"], + "MICRONAUT_ENVIRONMENTS" to "test", + "SERVICE_NAME" to project.name, + )) +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java index 36ac925fef1..f9011d78bb9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectionApiController.java @@ -160,8 +160,8 @@ public ConnectionRead getConnection(@Body final ConnectionIdRequestBody connecti @Secured({READER, WORKSPACE_READER, ORGANIZATION_READER}) @SecuredWorkspace @ExecuteOn(AirbyteTaskExecutors.IO) - public List getConnectionDataHistory(ConnectionDataHistoryRequestBody connectionDataHistoryRequestBody) { - return null; + public List getConnectionDataHistory(@Body final ConnectionDataHistoryRequestBody connectionDataHistoryRequestBody) { + return ApiHelper.execute(() -> connectionsHandler.getConnectionDataHistory(connectionDataHistoryRequestBody)); } @Override @@ -173,12 +173,14 @@ public List getConnectionStatuses(@Body final ConnectionSt return ApiHelper.execute(() -> connectionsHandler.getConnectionStatuses(connectionStatusesRequestBody)); } + @SuppressWarnings("LineLength") @Override @Post(uri = "/stream_history") @Secured({READER, WORKSPACE_READER, ORGANIZATION_READER}) @SecuredWorkspace @ExecuteOn(AirbyteTaskExecutors.IO) - public List getConnectionStreamHistory(ConnectionStreamHistoryRequestBody connectionStreamHistoryRequestBody) { + public List getConnectionStreamHistory( + @Body final ConnectionStreamHistoryRequestBody connectionStreamHistoryRequestBody) { return null; } @@ -187,7 +189,7 @@ public List getConnectionStreamHistory(Connecti @Secured({READER, WORKSPACE_READER, ORGANIZATION_READER}) @SecuredWorkspace @ExecuteOn(AirbyteTaskExecutors.IO) - public List getConnectionSyncProgress(ConnectionIdRequestBody connectionIdRequestBody) { + public List getConnectionSyncProgress(final ConnectionIdRequestBody connectionIdRequestBody) { return null; } @@ -196,7 +198,7 @@ public List getConnectionSyncProgress(Connection @Secured({READER, WORKSPACE_READER, ORGANIZATION_READER}) @SecuredWorkspace @ExecuteOn(AirbyteTaskExecutors.IO) - public List getConnectionUptimeHistory(ConnectionUptimeHistoryRequestBody connectionUptimeHistoryRequestBody) { + public List getConnectionUptimeHistory(final ConnectionUptimeHistoryRequestBody connectionUptimeHistoryRequestBody) { return null; } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectorBuilderProjectApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectorBuilderProjectApiController.java index 3fa291ee69c..9468bacfb28 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectorBuilderProjectApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConnectorBuilderProjectApiController.java @@ -6,7 +6,10 @@ import static io.airbyte.commons.auth.AuthRoleConstants.EDITOR; import static io.airbyte.commons.auth.AuthRoleConstants.ORGANIZATION_EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.ORGANIZATION_READER; +import static io.airbyte.commons.auth.AuthRoleConstants.READER; import static io.airbyte.commons.auth.AuthRoleConstants.WORKSPACE_EDITOR; +import static io.airbyte.commons.auth.AuthRoleConstants.WORKSPACE_READER; import io.airbyte.api.generated.ConnectorBuilderProjectApi; import io.airbyte.api.model.generated.ConnectorBuilderProjectIdWithWorkspaceId; @@ -67,7 +70,7 @@ public void deleteConnectorBuilderProject(final ConnectorBuilderProjectIdWithWor @Override @Post(uri = "/get_with_manifest") @Status(HttpStatus.OK) - @Secured({EDITOR, WORKSPACE_EDITOR, ORGANIZATION_EDITOR}) + @Secured({READER, WORKSPACE_READER, ORGANIZATION_READER}) @SecuredWorkspace @ExecuteOn(AirbyteTaskExecutors.IO) public ConnectorBuilderProjectRead getConnectorBuilderProject( @@ -78,7 +81,7 @@ public ConnectorBuilderProjectRead getConnectorBuilderProject( @Override @Post(uri = "/list") @Status(HttpStatus.OK) - @Secured({EDITOR, WORKSPACE_EDITOR, ORGANIZATION_EDITOR}) + @Secured({READER, WORKSPACE_READER, ORGANIZATION_READER}) @SecuredWorkspace @ExecuteOn(AirbyteTaskExecutors.IO) public ConnectorBuilderProjectReadList listConnectorBuilderProjects(final WorkspaceIdRequestBody workspaceIdRequestBody) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java index 57677f56ced..22a777f5e13 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java @@ -66,7 +66,7 @@ public ConnectionStateType getStateType(final ConnectionIdRequestBody connection } @Post("/check_updates") - @Secured({READER, WORKSPACE_READER, ORGANIZATION_READER}) + @Secured({AUTHENTICATED_USER}) @ExecuteOn(AirbyteTaskExecutors.IO) @Override public WebBackendCheckUpdatesRead webBackendCheckUpdates() { diff --git a/airbyte-temporal/build.gradle b/airbyte-temporal/build.gradle deleted file mode 100644 index 904e298d3c5..00000000000 --- a/airbyte-temporal/build.gradle +++ /dev/null @@ -1,20 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.docker" - id "io.airbyte.gradle.publish" -} - -airbyte { - docker { - imageName = "temporal" - } -} - -def copyScripts = tasks.register("copyScripts", Copy) { - from('scripts') - into 'build/airbyte/docker/' -} - -tasks.named("dockerBuildImage") { - dependsOn(copyScripts) -} diff --git a/airbyte-temporal/build.gradle.kts b/airbyte-temporal/build.gradle.kts new file mode 100644 index 00000000000..d699ef53873 --- /dev/null +++ b/airbyte-temporal/build.gradle.kts @@ -0,0 +1,20 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +airbyte { + docker { + imageName = "temporal" + } +} + +val copyScripts = tasks.register("copyScripts") { + from("scripts") + into("build/airbyte/docker/") +} + +tasks.named("dockerBuildImage") { + dependsOn(copyScripts) +} diff --git a/airbyte-test-utils/build.gradle b/airbyte-test-utils/build.gradle deleted file mode 100644 index d9e3c6b82cc..00000000000 --- a/airbyte-test-utils/build.gradle +++ /dev/null @@ -1,38 +0,0 @@ -plugins { - id "io.airbyte.gradle.jvm.lib" - id "io.airbyte.gradle.publish" -} - -configurations.all { - exclude group: 'io.micronaut.jaxrs' - exclude group: 'io.micronaut.sql' - - resolutionStrategy { - // Force to avoid updated version brought in transitively from Micronaut - force libs.platform.testcontainers.postgresql - } -} - -dependencies { - api project(':airbyte-db:db-lib') - implementation project(':airbyte-commons') - implementation project(':airbyte-api') - implementation project(':airbyte-commons-temporal') - implementation project(':airbyte-commons-worker') - - implementation libs.bundles.kubernetes.client - implementation libs.temporal.sdk - - api libs.junit.jupiter.api - - // Mark as compile only to avoid leaking transitively to connectors - compileOnly libs.platform.testcontainers.postgresql - - testImplementation libs.platform.testcontainers.postgresql - - testRuntimeOnly libs.junit.jupiter.engine - testImplementation libs.bundles.junit - testImplementation libs.assertj.core - - testImplementation libs.junit.pioneer -} diff --git a/airbyte-test-utils/build.gradle.kts b/airbyte-test-utils/build.gradle.kts new file mode 100644 index 00000000000..182f209f199 --- /dev/null +++ b/airbyte-test-utils/build.gradle.kts @@ -0,0 +1,38 @@ +plugins { + id("io.airbyte.gradle.jvm.lib") + id("io.airbyte.gradle.publish") +} + +configurations.all { + exclude( group = "io.micronaut.jaxrs") + exclude( group = "io.micronaut.sql") + + resolutionStrategy { + // Force to avoid(updated version(brought in transitively from Micronaut) + force(libs.platform.testcontainers.postgresql) + } +} + +dependencies { + api(project(":airbyte-db:db-lib")) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-api")) + implementation(project(":airbyte-commons-temporal")) + implementation(project(":airbyte-commons-worker")) + + implementation(libs.bundles.kubernetes.client) + implementation(libs.temporal.sdk) + + api(libs.junit.jupiter.api) + + // Mark as compile only(to avoid leaking transitively to connectors + compileOnly(libs.platform.testcontainers.postgresql) + + testImplementation(libs.platform.testcontainers.postgresql) + + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.bundles.junit) + testImplementation(libs.assertj.core) + + testImplementation(libs.junit.pioneer) +} diff --git a/airbyte-webapp/.eslintLegacyFolderStructure.js b/airbyte-webapp/.eslintLegacyFolderStructure.js index 5bde29c2e6f..9be1c6a99a0 100644 --- a/airbyte-webapp/.eslintLegacyFolderStructure.js +++ b/airbyte-webapp/.eslintLegacyFolderStructure.js @@ -1,14 +1,11 @@ // A list of all files that existed still in the old folder structure module.exports = [ // src/services - "src/services/connector/SourceDefinitionService.ts", - "src/services/connector/DestinationDefinitionService.ts", "src/services/connectorBuilder/ConnectorBuilderTestInputService.tsx", "src/services/connectorBuilder/ConnectorBuilderStateService.tsx", "src/services/connectorBuilder/ConnectorBuilderLocalStorageService.tsx", "src/services/connectorBuilder/SchemaWorker.ts", "src/services/useInitService.tsx", - "src/services/useDefaultRequestMiddlewares.tsx", "src/services/Scope.ts", // src/hooks "src/hooks/useDeleteModal.tsx", @@ -21,7 +18,6 @@ module.exports = [ "src/hooks/services/Health/index.tsx", "src/hooks/services/useRequestConnector.tsx", "src/hooks/services/ConnectionEdit/ConnectionEditService.tsx", - "src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx", "src/hooks/services/ConfirmationModal/types.ts", "src/hooks/services/ConfirmationModal/ConfirmationModalService.tsx", "src/hooks/services/ConfirmationModal/index.ts", @@ -35,7 +31,6 @@ module.exports = [ "src/hooks/services/Modal/ModalService.test.tsx", "src/hooks/services/Modal/index.ts", "src/hooks/services/useRequestErrorHandler.tsx", - "src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx", "src/hooks/services/ConnectionForm/ConnectionFormService.tsx", "src/hooks/services/useConnectorAuthRevocation.tsx", "src/hooks/services/Experiment/experiments.ts", @@ -128,16 +123,8 @@ module.exports = [ "src/views/layout/MainView/MainView.tsx", "src/views/layout/MainView/index.tsx", "src/views/layout/SideBar/components/ResourcesDropdown.tsx", - "src/views/layout/SideBar/components/BuilderIcon.tsx", "src/views/layout/SideBar/components/NavDropdown.tsx", - "src/views/layout/SideBar/components/ConnectionsIcon.tsx", - "src/views/layout/SideBar/components/ChatIcon.tsx", - "src/views/layout/SideBar/components/StatusIcon.tsx", "src/views/layout/SideBar/components/MenuContent.tsx", - "src/views/layout/SideBar/components/DestinationIcon.tsx", - "src/views/layout/SideBar/components/SettingsIcon.tsx", - "src/views/layout/SideBar/components/RecipesIcon.tsx", - "src/views/layout/SideBar/components/SourceIcon.tsx", "src/views/layout/SideBar/components/NavItem.tsx", "src/views/layout/SideBar/NotificationIndicator.tsx", "src/views/layout/SideBar/AirbyteHomeLink.tsx", diff --git a/airbyte-webapp/.eslintrc.js b/airbyte-webapp/.eslintrc.js index c3da5e511e5..f60ea32af5e 100644 --- a/airbyte-webapp/.eslintrc.js +++ b/airbyte-webapp/.eslintrc.js @@ -133,6 +133,7 @@ module.exports = { "prefer-template": "warn", "spaced-comment": ["warn", "always", { markers: ["/"] }], yoda: "warn", + "import/no-duplicates": ["warn", { considerQueryString: true }], "import/order": [ "warn", { diff --git a/airbyte-webapp/cypress/commands/api/payloads.ts b/airbyte-webapp/cypress/commands/api/payloads.ts index 791b87a534d..d8b1381ee3a 100644 --- a/airbyte-webapp/cypress/commands/api/payloads.ts +++ b/airbyte-webapp/cypress/commands/api/payloads.ts @@ -1,6 +1,10 @@ import { ConnectorIds } from "@src/area/connector/utils/constants"; -import { WebBackendConnectionCreate } from "@src/core/api/types/AirbyteClient"; -import { DestinationCreate, SourceCreate, WebBackendConnectionUpdate } from "@src/core/api/types/AirbyteClient"; +import { + WebBackendConnectionCreate, + DestinationCreate, + SourceCreate, + WebBackendConnectionUpdate, +} from "@src/core/api/types/AirbyteClient"; import { getWorkspaceId } from "./workspace"; diff --git a/airbyte-webapp/cypress/e2e/connection/configuration.cy.ts b/airbyte-webapp/cypress/e2e/connection/configuration.cy.ts index fe300d14f97..f35530ef289 100644 --- a/airbyte-webapp/cypress/e2e/connection/configuration.cy.ts +++ b/airbyte-webapp/cypress/e2e/connection/configuration.cy.ts @@ -24,8 +24,7 @@ import { waitForUpdateConnectionRequest, } from "@cy/commands/interceptors"; import * as connectionForm from "@cy/pages/connection/connectionFormPageObject"; -import { getSyncEnabledSwitch } from "@cy/pages/connection/connectionPageObject"; -import { visit } from "@cy/pages/connection/connectionPageObject"; +import { getSyncEnabledSwitch, visit } from "@cy/pages/connection/connectionPageObject"; import * as replicationPage from "@cy/pages/connection/connectionReplicationPageObject"; import { streamsTable } from "@cy/pages/connection/StreamsTablePageObject"; import { @@ -493,7 +492,7 @@ describe("Connection Configuration", () => { it("Cannot edit fields in Configuration section", () => { cy.get("@connection").then((connection) => { cy.visit(`/${RoutePaths.Connections}/${connection.connectionId}/${ConnectionRoutePaths.Replication}`); - cy.get(connectionForm.scheduleDropdown).within(() => cy.get("input").should("be.disabled")); + cy.get(connectionForm.scheduleTypeDropdown).within(() => cy.get("input").should("be.disabled")); cy.get(connectionForm.destinationNamespaceEditButton).should("be.disabled"); cy.get(connectionForm.destinationPrefixEditButton).should("be.disabled"); cy.get(replicationPage.nonBreakingChangesPreference).within(() => cy.get("input").should("be.disabled")); diff --git a/airbyte-webapp/cypress/e2e/connection/createConnection.cy.ts b/airbyte-webapp/cypress/e2e/connection/createConnection.cy.ts index 47526828942..1af2d5f394a 100644 --- a/airbyte-webapp/cypress/e2e/connection/createConnection.cy.ts +++ b/airbyte-webapp/cypress/e2e/connection/createConnection.cy.ts @@ -3,11 +3,9 @@ import { createPostgresDestinationViaApi, createPostgresSourceViaApi, } from "@cy/commands/connection"; -import { fillLocalJsonForm } from "@cy/commands/connector"; -import { fillPokeAPIForm } from "@cy/commands/connector"; +import { fillLocalJsonForm, fillPokeAPIForm } from "@cy/commands/connector"; import { goToDestinationPage, openDestinationConnectionsPage } from "@cy/pages/destinationPage"; -import { openSourceConnectionsPage } from "@cy/pages/sourcePage"; -import { goToSourcePage } from "@cy/pages/sourcePage"; +import { openSourceConnectionsPage, goToSourcePage } from "@cy/pages/sourcePage"; import { WebBackendConnectionRead, DestinationRead, SourceRead } from "@src/core/api/types/AirbyteClient"; import { RoutePaths, ConnectionRoutePaths } from "@src/pages/routePaths"; import { requestDeleteConnection, requestDeleteDestination, requestDeleteSource } from "commands/api"; diff --git a/airbyte-webapp/cypress/pages/connection/connectionFormPageObject.ts b/airbyte-webapp/cypress/pages/connection/connectionFormPageObject.ts index 4435a9b219d..a425ac946d6 100644 --- a/airbyte-webapp/cypress/pages/connection/connectionFormPageObject.ts +++ b/airbyte-webapp/cypress/pages/connection/connectionFormPageObject.ts @@ -2,7 +2,8 @@ import { getTestId } from "utils/selectors"; const connectionNameInput = getTestId("connectionName"); const expandConfigurationIcon = getTestId("configuration-card-expand-arrow"); -export const scheduleDropdown = getTestId("scheduleData"); +export const scheduleTypeDropdown = getTestId("schedule-type-listbox-button"); +export const getScheduleTypeDropdownOption = (option: string) => `${getTestId(`${option.toLowerCase()}-option`)}`; export const destinationPrefixEditButton = getTestId("destination-stream-prefix-edit-button"); const destinationPrefixApplyButton = getTestId("destination-stream-names-apply-button"); @@ -24,8 +25,8 @@ export const enterConnectionName = (name: string) => { }; export const selectSchedule = (value: string) => { - cy.get(scheduleDropdown).click(); - cy.get(getTestId(value)).click(); + cy.get(scheduleTypeDropdown).click(); + cy.get(getScheduleTypeDropdownOption(value)).click(); }; export const expandConfigurationSection = () => { diff --git a/airbyte-webapp/cypress/pages/connection/connectionReplicationPageObject.ts b/airbyte-webapp/cypress/pages/connection/connectionReplicationPageObject.ts index 3d679e9c689..fd280a8b19c 100644 --- a/airbyte-webapp/cypress/pages/connection/connectionReplicationPageObject.ts +++ b/airbyte-webapp/cypress/pages/connection/connectionReplicationPageObject.ts @@ -12,7 +12,7 @@ const schemaChangesDetectedBanner = "[data-testid='schemaChangesDetected']"; const schemaChangesReviewButton = "[data-testid='schemaChangesDetected-button']"; const schemaChangesBackdrop = "[data-testid='schemaChangesBackdrop']"; export const nonBreakingChangesPreference = "[data-testid='nonBreakingChangesPreference']"; -const nonBreakingChangesPreferenceValue = (value: string) => `[data-testid='nonBreakingChangesPreference-${value}']`; +const nonBreakingChangesPreferenceValue = (value: string) => `[data-testid='${value}-option']`; const noDiffToast = "[data-testid='notification-connection.noDiff']"; const cancelButton = getTestId("cancel-edit-button", "button"); const saveButton = getTestId("save-edit-button", "button"); diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 92a5493d7e0..ed33a4e1465 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -61,11 +61,6 @@ "@dnd-kit/sortable": "^7.0.2", "@dnd-kit/utilities": "^3.2.1", "@floating-ui/react-dom": "^1.0.0", - "@fortawesome/fontawesome-svg-core": "^6.1.1", - "@fortawesome/free-brands-svg-icons": "^6.1.1", - "@fortawesome/free-regular-svg-icons": "^6.1.1", - "@fortawesome/free-solid-svg-icons": "^6.1.1", - "@fortawesome/react-fontawesome": "^0.1.18", "@headlessui-float/react": "^0.11.2", "@headlessui/react": "1.7.13", "@hookform/resolvers": "^2.9.11", @@ -89,7 +84,6 @@ "diff": "^5.1.0", "firebase": "^10.5.0", "flat": "^5.0.2", - "formik": "^2.2.9", "framer-motion": "^6.3.11", "js-yaml": "^4.1.0", "json-schema": "^0.4.0", diff --git a/airbyte-webapp/pnpm-lock.yaml b/airbyte-webapp/pnpm-lock.yaml index 5d5c3ddd330..299793e9fe8 100644 --- a/airbyte-webapp/pnpm-lock.yaml +++ b/airbyte-webapp/pnpm-lock.yaml @@ -33,21 +33,6 @@ dependencies: '@floating-ui/react-dom': specifier: ^1.0.0 version: 1.2.1(react-dom@18.2.0)(react@18.2.0) - '@fortawesome/fontawesome-svg-core': - specifier: ^6.1.1 - version: 6.2.1 - '@fortawesome/free-brands-svg-icons': - specifier: ^6.1.1 - version: 6.2.1 - '@fortawesome/free-regular-svg-icons': - specifier: ^6.1.1 - version: 6.2.1 - '@fortawesome/free-solid-svg-icons': - specifier: ^6.1.1 - version: 6.2.1 - '@fortawesome/react-fontawesome': - specifier: ^0.1.18 - version: 0.1.19(@fortawesome/fontawesome-svg-core@6.2.1)(react@18.2.0) '@headlessui-float/react': specifier: ^0.11.2 version: 0.11.2(@headlessui/react@1.7.13)(react-dom@18.2.0)(react@18.2.0) @@ -117,9 +102,6 @@ dependencies: flat: specifier: ^5.0.2 version: 5.0.2 - formik: - specifier: ^2.2.9 - version: 2.2.9(react@18.2.0) framer-motion: specifier: ^6.3.11 version: 6.5.1(react-dom@18.2.0)(react@18.2.0) @@ -3282,55 +3264,6 @@ packages: typescript: 5.0.2 dev: false - /@fortawesome/fontawesome-common-types@6.2.1: - resolution: {integrity: sha512-Sz07mnQrTekFWLz5BMjOzHl/+NooTdW8F8kDQxjWwbpOJcnoSg4vUDng8d/WR1wOxM0O+CY9Zw0nR054riNYtQ==} - engines: {node: '>=6'} - requiresBuild: true - dev: false - - /@fortawesome/fontawesome-svg-core@6.2.1: - resolution: {integrity: sha512-HELwwbCz6C1XEcjzyT1Jugmz2NNklMrSPjZOWMlc+ZsHIVk+XOvOXLGGQtFBwSyqfJDNgRq4xBCwWOaZ/d9DEA==} - engines: {node: '>=6'} - requiresBuild: true - dependencies: - '@fortawesome/fontawesome-common-types': 6.2.1 - dev: false - - /@fortawesome/free-brands-svg-icons@6.2.1: - resolution: {integrity: sha512-L8l4MfdHPmZlJ72PvzdfwOwbwcCAL0vx48tJRnI6u1PJXh+j2f3yDoKyQgO3qjEsgD5Fr2tQV/cPP8F/k6aUig==} - engines: {node: '>=6'} - requiresBuild: true - dependencies: - '@fortawesome/fontawesome-common-types': 6.2.1 - dev: false - - /@fortawesome/free-regular-svg-icons@6.2.1: - resolution: {integrity: sha512-wiqcNDNom75x+pe88FclpKz7aOSqS2lOivZeicMV5KRwOAeypxEYWAK/0v+7r+LrEY30+qzh8r2XDaEHvoLsMA==} - engines: {node: '>=6'} - requiresBuild: true - dependencies: - '@fortawesome/fontawesome-common-types': 6.2.1 - dev: false - - /@fortawesome/free-solid-svg-icons@6.2.1: - resolution: {integrity: sha512-oKuqrP5jbfEPJWTij4sM+/RvgX+RMFwx3QZCZcK9PrBDgxC35zuc7AOFsyMjMd/PIFPeB2JxyqDr5zs/DZFPPw==} - engines: {node: '>=6'} - requiresBuild: true - dependencies: - '@fortawesome/fontawesome-common-types': 6.2.1 - dev: false - - /@fortawesome/react-fontawesome@0.1.19(@fortawesome/fontawesome-svg-core@6.2.1)(react@18.2.0): - resolution: {integrity: sha512-Hyb+lB8T18cvLNX0S3llz7PcSOAJMLwiVKBuuzwM/nI5uoBw+gQjnf9il0fR1C3DKOI5Kc79pkJ4/xB0Uw9aFQ==} - peerDependencies: - '@fortawesome/fontawesome-svg-core': ~1 || ~6 - react: '>=16.x' - dependencies: - '@fortawesome/fontawesome-svg-core': 6.2.1 - prop-types: 15.8.1 - react: 18.2.0 - dev: false - /@grpc/grpc-js@1.9.6: resolution: {integrity: sha512-yq3qTy23u++8zdvf+h4mz4ohDFi681JAkMZZPTKh8zmUVh0AKLisFlgxcn22FMNowXz15oJ6pqgwT7DJ+PdJvg==} engines: {node: ^8.13.0 || >=10.10.0} @@ -9033,6 +8966,7 @@ packages: /deepmerge@2.2.1: resolution: {integrity: sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA==} engines: {node: '>=0.10.0'} + dev: true /deepmerge@4.3.1: resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} @@ -10704,21 +10638,6 @@ packages: engines: {node: '>=0.4.x'} dev: false - /formik@2.2.9(react@18.2.0): - resolution: {integrity: sha512-LQLcISMmf1r5at4/gyJigGn0gOwFbeEAlji+N9InZF6LIMXnFNkO42sCI8Jt84YZggpD4cPWObAZaxpEFtSzNA==} - peerDependencies: - react: '>=16.8.0' - dependencies: - deepmerge: 2.2.1 - hoist-non-react-statics: 3.3.2 - lodash: 4.17.21 - lodash-es: 4.17.21 - react: 18.2.0 - react-fast-compare: 2.0.4 - tiny-warning: 1.0.3 - tslib: 1.14.1 - dev: false - /forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -15062,10 +14981,6 @@ packages: react-is: 18.1.0 dev: true - /react-fast-compare@2.0.4: - resolution: {integrity: sha512-suNP+J1VU1MWFKcyt7RtjiSWUjvidmQSlqu+eHslq+342xCbGTYmC0mEhPCOHxlW0CywylOC1u2DFAT+bv4dBw==} - dev: false - /react-fast-compare@3.2.0: resolution: {integrity: sha512-rtGImPZ0YyLrscKI9xTpV8psd6I8VAtjKCzQDlzyDvqJA8XOW78TXYQwNRNd8g8JZnDu8q9Fu/1v4HPAVwVdHA==} @@ -17173,10 +17088,6 @@ packages: resolution: {integrity: sha512-AD5ih2NlSssTCwsMznbvwMZpJ1cbhkGd2uueNxzv2jDlEeZdU04JQfRnggJQ8DrcVBGjAsCKwFBbDlVNtEMlzw==} dev: true - /tiny-warning@1.0.3: - resolution: {integrity: sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==} - dev: false - /titleize@3.0.0: resolution: {integrity: sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ==} engines: {node: '>=12'} @@ -17358,6 +17269,7 @@ packages: /tslib@1.14.1: resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==} + dev: true /tslib@2.3.1: resolution: {integrity: sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==} diff --git a/airbyte-webapp/src/App.tsx b/airbyte-webapp/src/App.tsx index 10233d3dd84..e403c9c3177 100644 --- a/airbyte-webapp/src/App.tsx +++ b/airbyte-webapp/src/App.tsx @@ -6,7 +6,7 @@ import { ThemeProvider } from "styled-components"; import { ApiErrorBoundary } from "components/common/ApiErrorBoundary"; import { DevToolsToggle } from "components/DevToolsToggle"; -import { config } from "config"; +import { config, ConfigServiceProvider } from "config"; import { QueryProvider, useGetInstanceConfiguration } from "core/api"; import { AnalyticsProvider } from "core/services/analytics"; import { OSSAuthService } from "core/services/auth"; @@ -23,7 +23,6 @@ import { AirbyteThemeProvider } from "hooks/theme/useAirbyteTheme"; import { ConnectorBuilderTestInputProvider } from "services/connectorBuilder/ConnectorBuilderTestInputService"; import LoadingPage from "./components/LoadingPage"; -import { ConfigServiceProvider } from "./config"; import en from "./locales/en.json"; import { Routing } from "./pages/routes"; import { theme } from "./theme"; @@ -34,8 +33,8 @@ const StyleProvider: React.FC> = ({ children }) const Services: React.FC> = ({ children }) => ( - - + + @@ -45,8 +44,8 @@ const Services: React.FC> = ({ children }) => ( - - + + ); diff --git a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss b/airbyte-webapp/src/area/connection/components/AttemptDetails/AttemptDetails.module.scss similarity index 100% rename from airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss rename to airbyte-webapp/src/area/connection/components/AttemptDetails/AttemptDetails.module.scss diff --git a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx b/airbyte-webapp/src/area/connection/components/AttemptDetails/AttemptDetails.tsx similarity index 92% rename from airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx rename to airbyte-webapp/src/area/connection/components/AttemptDetails/AttemptDetails.tsx index 14e954e1489..147294089a6 100644 --- a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx +++ b/airbyte-webapp/src/area/connection/components/AttemptDetails/AttemptDetails.tsx @@ -5,11 +5,15 @@ import { FormattedDate, FormattedMessage, FormattedTimeParts, useIntl } from "re import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; -import { AttemptRead, AttemptStatus } from "core/request/AirbyteClient"; +import { AttemptRead, AttemptStatus, FailureReason, FailureType } from "core/request/AirbyteClient"; import { formatBytes } from "core/utils/numberHelper"; import styles from "./AttemptDetails.module.scss"; -import { getFailureFromAttempt, isCancelledAttempt } from "../utils"; + +const getFailureFromAttempt = (attempt: AttemptRead): FailureReason | undefined => attempt.failureSummary?.failures[0]; + +const isCancelledAttempt = (attempt: AttemptRead): boolean => + attempt.failureSummary?.failures.some(({ failureType }) => failureType === FailureType.manual_cancellation) ?? false; interface AttemptDetailsProps { className?: string; diff --git a/airbyte-webapp/src/area/connection/components/AttemptDetails/index.ts b/airbyte-webapp/src/area/connection/components/AttemptDetails/index.ts new file mode 100644 index 00000000000..90c22486ab0 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/AttemptDetails/index.ts @@ -0,0 +1 @@ +export * from "./AttemptDetails"; diff --git a/airbyte-webapp/src/components/NewJobItem/NewJobItem.module.scss b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobHistoryItem.module.scss similarity index 94% rename from airbyte-webapp/src/components/NewJobItem/NewJobItem.module.scss rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/JobHistoryItem.module.scss index f5c82db3103..8138cf2246f 100644 --- a/airbyte-webapp/src/components/NewJobItem/NewJobItem.module.scss +++ b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobHistoryItem.module.scss @@ -1,7 +1,7 @@ @use "scss/colors"; @use "scss/variables"; -.newJobItem { +.jobHistoryItem { border: variables.$border-thick solid transparent; border-top: variables.$border-thin solid colors.$grey-100; padding: variables.$spacing-xl; @@ -19,7 +19,7 @@ // fix for "dancing" scrollbar &__spinnerContainer { - overflow: clip + overflow: clip; } &__summary { diff --git a/airbyte-webapp/src/components/NewJobItem/NewJobItem.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobHistoryItem.tsx similarity index 83% rename from airbyte-webapp/src/components/NewJobItem/NewJobItem.tsx rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/JobHistoryItem.tsx index 3487e18224a..477e75d6fa3 100644 --- a/airbyte-webapp/src/components/NewJobItem/NewJobItem.tsx +++ b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobHistoryItem.tsx @@ -1,24 +1,23 @@ -import { faEllipsisV } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import { Suspense, useCallback, useRef } from "react"; import { FormattedDate, FormattedMessage, FormattedTimeParts, useIntl } from "react-intl"; import { useEffectOnce } from "react-use"; -import { buildAttemptLink, useAttemptLink } from "components/JobItem/attemptLinkUtils"; -import { AttemptDetails } from "components/JobItem/components/AttemptDetails"; -import { getJobCreatedAt } from "components/JobItem/components/JobSummary"; -import { ResetStreamsDetails } from "components/JobItem/components/ResetStreamDetails"; -import { JobWithAttempts } from "components/JobItem/types"; -import { getJobAttempts } from "components/JobItem/utils"; import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; import { DropdownMenu, DropdownMenuOptionType } from "components/ui/DropdownMenu"; import { FlexContainer } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { LoadingSpinner } from "components/ui/LoadingSpinner"; import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; +import { AttemptDetails } from "area/connection/components/AttemptDetails"; +import { ResetStreamsDetails } from "area/connection/components/JobHistoryItem/ResetStreamDetails"; +import { JobLogsModal } from "area/connection/components/JobLogsModal/JobLogsModal"; +import { JobWithAttempts } from "area/connection/types/jobs"; +import { buildAttemptLink, useAttemptLink } from "area/connection/utils/attemptLink"; +import { isJobPartialSuccess, getJobAttempts, getJobCreatedAt } from "area/connection/utils/jobs"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useCurrentWorkspace, useGetDebugInfoJobManual } from "core/api"; import { copyToClipboard } from "core/utils/clipboard"; @@ -28,13 +27,11 @@ import { useConnectionEditService } from "hooks/services/ConnectionEdit/Connecti import { useModalService } from "hooks/services/Modal"; import { useNotificationService } from "hooks/services/Notification"; -import { isPartialSuccess } from "./isPartialSuccess"; -import { JobLogsModalContent } from "./JobLogsModalContent"; +import styles from "./JobHistoryItem.module.scss"; import { JobStatusIcon } from "./JobStatusIcon"; import { JobStatusLabel } from "./JobStatusLabel"; -import styles from "./NewJobItem.module.scss"; -interface NewJobItemProps { +interface JobHistoryItemProps { jobWithAttempts: JobWithAttempts; } @@ -44,7 +41,7 @@ enum ContextMenuOptions { DownloadLogs = "DownloadLogs", } -export const NewJobItem: React.FC = ({ jobWithAttempts }) => { +export const JobHistoryItem: React.FC = ({ jobWithAttempts }) => { const { openModal } = useModalService(); const attempts = getJobAttempts(jobWithAttempts); const attemptLink = useAttemptLink(); @@ -72,12 +69,12 @@ export const NewJobItem: React.FC = ({ jobWithAttempts }) => { content: () => ( +

} > - + ), }); @@ -94,7 +91,7 @@ export const NewJobItem: React.FC = ({ jobWithAttempts }) => { text: ( -
+
@@ -165,16 +162,16 @@ export const NewJobItem: React.FC = ({ jobWithAttempts }) => { return (
- - + + {jobWithAttempts.job.configType === "reset_connection" ? ( = ({ jobWithAttempts }) => { attempt={attempts[attempts.length - 1]} hasMultipleAttempts={attempts.length > 1} jobId={String(jobWithAttempts.job.id)} - isPartialSuccess={isPartialSuccess(jobWithAttempts.attempts)} + isPartialSuccess={isJobPartialSuccess(jobWithAttempts.attempts)} /> ) )} - + {(parts) => {`${parts[0].value}:${parts[2].value}${parts[4].value} `}} @@ -233,7 +230,7 @@ export const NewJobItem: React.FC = ({ jobWithAttempts }) => { ]} onChange={handleClick} > - {() =>
); diff --git a/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusIcon.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusIcon.tsx new file mode 100644 index 00000000000..bcc800752b8 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusIcon.tsx @@ -0,0 +1,28 @@ +import { StatusIcon } from "components/ui/StatusIcon"; + +import { JobWithAttempts } from "area/connection/types/jobs"; +import { isJobPartialSuccess, didJobSucceed, getJobStatus } from "area/connection/utils/jobs"; +import { JobStatus } from "core/request/AirbyteClient"; + +interface JobStatusIconProps { + job: JobWithAttempts; +} + +export const JobStatusIcon: React.FC = ({ job }) => { + const didSucceed = didJobSucceed(job); + const jobStatus = getJobStatus(job); + const jobIsPartialSuccess = isJobPartialSuccess(job.attempts); + + if (jobIsPartialSuccess) { + return ; + } else if (!didSucceed) { + return ; + } else if (jobStatus === JobStatus.cancelled) { + return ; + } else if (jobStatus === JobStatus.running || jobStatus === JobStatus.incomplete) { + return ; + } else if (jobStatus === JobStatus.succeeded) { + return ; + } + return null; +}; diff --git a/airbyte-webapp/src/components/NewJobItem/JobStatusLabel.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusLabel.tsx similarity index 83% rename from airbyte-webapp/src/components/NewJobItem/JobStatusLabel.tsx rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusLabel.tsx index 2b16c84c455..1004f1f71c4 100644 --- a/airbyte-webapp/src/components/NewJobItem/JobStatusLabel.tsx +++ b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusLabel.tsx @@ -1,13 +1,11 @@ import { FormattedMessage } from "react-intl"; -import { JobWithAttempts } from "components/JobItem/types"; -import { getJobAttempts, getJobStatus } from "components/JobItem/utils"; import { Text } from "components/ui/Text"; +import { JobWithAttempts } from "area/connection/types/jobs"; +import { isJobPartialSuccess, getJobAttempts, getJobStatus } from "area/connection/utils/jobs"; import { JobStatus } from "core/request/AirbyteClient"; -import { isPartialSuccess } from "./isPartialSuccess"; - interface JobStatusLabelProps { jobWithAttempts: JobWithAttempts; } @@ -15,7 +13,7 @@ interface JobStatusLabelProps { export const JobStatusLabel: React.FC = ({ jobWithAttempts }) => { const attempts = getJobAttempts(jobWithAttempts); const jobStatus = getJobStatus(jobWithAttempts); - const jobIsPartialSuccess = isPartialSuccess(attempts); + const jobIsPartialSuccess = isJobPartialSuccess(attempts); const streamsToReset = "job" in jobWithAttempts ? jobWithAttempts.job.resetConfig?.streamsToReset : undefined; const jobConfigType = jobWithAttempts.job.configType; diff --git a/airbyte-webapp/src/components/NewJobItem/LogSearchInput.module.scss b/airbyte-webapp/src/area/connection/components/JobHistoryItem/LogSearchInput.module.scss similarity index 100% rename from airbyte-webapp/src/components/NewJobItem/LogSearchInput.module.scss rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/LogSearchInput.module.scss diff --git a/airbyte-webapp/src/components/NewJobItem/LogSearchInput.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/LogSearchInput.tsx similarity index 100% rename from airbyte-webapp/src/components/NewJobItem/LogSearchInput.tsx rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/LogSearchInput.tsx diff --git a/airbyte-webapp/src/components/JobItem/components/ResetStreamDetails.module.scss b/airbyte-webapp/src/area/connection/components/JobHistoryItem/ResetStreamDetails.module.scss similarity index 89% rename from airbyte-webapp/src/components/JobItem/components/ResetStreamDetails.module.scss rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/ResetStreamDetails.module.scss index 4a8f2ef8538..0f5bfb2c3d0 100644 --- a/airbyte-webapp/src/components/JobItem/components/ResetStreamDetails.module.scss +++ b/airbyte-webapp/src/area/connection/components/JobHistoryItem/ResetStreamDetails.module.scss @@ -1,5 +1,5 @@ -@use "../../../scss/colors"; -@use "../../../scss/variables"; +@use "scss/colors"; +@use "scss/variables"; .textContainer { margin: variables.$spacing-xs 0 0; diff --git a/airbyte-webapp/src/components/JobItem/components/ResetStreamDetails.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/ResetStreamDetails.tsx similarity index 100% rename from airbyte-webapp/src/components/JobItem/components/ResetStreamDetails.tsx rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/ResetStreamDetails.tsx diff --git a/airbyte-webapp/src/components/NewJobItem/VirtualLogs.module.scss b/airbyte-webapp/src/area/connection/components/JobHistoryItem/VirtualLogs.module.scss similarity index 100% rename from airbyte-webapp/src/components/NewJobItem/VirtualLogs.module.scss rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/VirtualLogs.module.scss diff --git a/airbyte-webapp/src/components/NewJobItem/VirtualLogs.test.ts b/airbyte-webapp/src/area/connection/components/JobHistoryItem/VirtualLogs.test.ts similarity index 100% rename from airbyte-webapp/src/components/NewJobItem/VirtualLogs.test.ts rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/VirtualLogs.test.ts diff --git a/airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/VirtualLogs.tsx similarity index 100% rename from airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/VirtualLogs.tsx diff --git a/airbyte-webapp/src/area/connection/components/JobHistoryItem/index.ts b/airbyte-webapp/src/area/connection/components/JobHistoryItem/index.ts new file mode 100644 index 00000000000..0f7fe3e9f34 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/JobHistoryItem/index.ts @@ -0,0 +1 @@ +export { JobHistoryItem } from "./JobHistoryItem"; diff --git a/airbyte-webapp/src/components/NewJobItem/useCleanLogs.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/useCleanLogs.tsx similarity index 100% rename from airbyte-webapp/src/components/NewJobItem/useCleanLogs.tsx rename to airbyte-webapp/src/area/connection/components/JobHistoryItem/useCleanLogs.tsx diff --git a/airbyte-webapp/src/area/connection/components/JobLogsModal/AttemptStatusIcon.tsx b/airbyte-webapp/src/area/connection/components/JobLogsModal/AttemptStatusIcon.tsx new file mode 100644 index 00000000000..5474980710b --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/JobLogsModal/AttemptStatusIcon.tsx @@ -0,0 +1,18 @@ +import { StatusIcon } from "components/ui/StatusIcon"; + +import { AttemptInfoRead, AttemptStatus } from "core/request/AirbyteClient"; + +interface AttemptStatusIconProps { + attempt: AttemptInfoRead; +} + +export const AttemptStatusIcon: React.FC = ({ attempt }) => { + if (attempt.attempt.status === AttemptStatus.failed) { + return ; + } else if (attempt.attempt.status === AttemptStatus.running) { + return ; + } else if (attempt.attempt.status === AttemptStatus.succeeded) { + return ; + } + return null; +}; diff --git a/airbyte-webapp/src/components/NewJobItem/DownloadLogsButton.tsx b/airbyte-webapp/src/area/connection/components/JobLogsModal/DownloadLogsButton.tsx similarity index 91% rename from airbyte-webapp/src/components/NewJobItem/DownloadLogsButton.tsx rename to airbyte-webapp/src/area/connection/components/JobLogsModal/DownloadLogsButton.tsx index e7735991f83..7bfd20f3bb0 100644 --- a/airbyte-webapp/src/components/NewJobItem/DownloadLogsButton.tsx +++ b/airbyte-webapp/src/area/connection/components/JobLogsModal/DownloadLogsButton.tsx @@ -4,11 +4,10 @@ import { useIntl } from "react-intl"; import { Button } from "components/ui/Button"; import { Icon } from "components/ui/Icon"; +import { CleanedLogLines } from "area/connection/components/JobHistoryItem/useCleanLogs"; import { useCurrentWorkspace } from "core/api"; import { FILE_TYPE_DOWNLOAD, downloadFile, fileizeString } from "core/utils/file"; -import { CleanedLogLines } from "./useCleanLogs"; - interface DownloadButtonProps { logLines: CleanedLogLines; fileName: string; diff --git a/airbyte-webapp/src/components/NewJobItem/JobLogsModalContent.module.scss b/airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModal.module.scss similarity index 100% rename from airbyte-webapp/src/components/NewJobItem/JobLogsModalContent.module.scss rename to airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModal.module.scss diff --git a/airbyte-webapp/src/components/NewJobItem/JobLogsModalContent.tsx b/airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModal.tsx similarity index 90% rename from airbyte-webapp/src/components/NewJobItem/JobLogsModalContent.tsx rename to airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModal.tsx index 900fb07b374..9b880eef1ac 100644 --- a/airbyte-webapp/src/components/NewJobItem/JobLogsModalContent.tsx +++ b/airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModal.tsx @@ -2,28 +2,28 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { useDebounce } from "react-use"; -import { AttemptDetails } from "components/JobItem/components/AttemptDetails"; -import { LinkToAttemptButton } from "components/JobItem/components/LinkToAttemptButton"; import { Box } from "components/ui/Box"; import { FlexContainer } from "components/ui/Flex"; import { ListBox } from "components/ui/ListBox"; +import { AttemptDetails } from "area/connection/components/AttemptDetails"; +import { LogSearchInput } from "area/connection/components/JobHistoryItem/LogSearchInput"; +import { useCleanLogs } from "area/connection/components/JobHistoryItem/useCleanLogs"; +import { VirtualLogs } from "area/connection/components/JobHistoryItem/VirtualLogs"; +import { LinkToAttemptButton } from "area/connection/components/JobLogsModal/LinkToAttemptButton"; import { useAttemptForJob, useJobInfoWithoutLogs } from "core/api"; +import { AttemptStatusIcon } from "./AttemptStatusIcon"; import { DownloadLogsButton } from "./DownloadLogsButton"; -import styles from "./JobLogsModalContent.module.scss"; +import styles from "./JobLogsModal.module.scss"; import { JobLogsModalFailureMessage } from "./JobLogsModalFailureMessage"; -import { AttemptStatusIcon } from "./JobStatusIcon"; -import { LogSearchInput } from "./LogSearchInput"; -import { useCleanLogs } from "./useCleanLogs"; -import { VirtualLogs } from "./VirtualLogs"; -interface JobLogsModalContentProps { +interface JobLogsModalProps { jobId: number; initialAttemptId?: number; } -export const JobLogsModalContent: React.FC = ({ jobId, initialAttemptId }) => { +export const JobLogsModal: React.FC = ({ jobId, initialAttemptId }) => { const searchInputRef = useRef(null); const [inputValue, setInputValue] = useState(""); const job = useJobInfoWithoutLogs(jobId); diff --git a/airbyte-webapp/src/components/NewJobItem/JobLogsModalFailureMessage.module.scss b/airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModalFailureMessage.module.scss similarity index 100% rename from airbyte-webapp/src/components/NewJobItem/JobLogsModalFailureMessage.module.scss rename to airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModalFailureMessage.module.scss diff --git a/airbyte-webapp/src/components/NewJobItem/JobLogsModalFailureMessage.tsx b/airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModalFailureMessage.tsx similarity index 100% rename from airbyte-webapp/src/components/NewJobItem/JobLogsModalFailureMessage.tsx rename to airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModalFailureMessage.tsx diff --git a/airbyte-webapp/src/components/JobItem/components/LinkToAttemptButton.tsx b/airbyte-webapp/src/area/connection/components/JobLogsModal/LinkToAttemptButton.tsx similarity index 85% rename from airbyte-webapp/src/components/JobItem/components/LinkToAttemptButton.tsx rename to airbyte-webapp/src/area/connection/components/JobLogsModal/LinkToAttemptButton.tsx index a89ba58847e..a8f7267538d 100644 --- a/airbyte-webapp/src/components/JobItem/components/LinkToAttemptButton.tsx +++ b/airbyte-webapp/src/area/connection/components/JobLogsModal/LinkToAttemptButton.tsx @@ -1,16 +1,14 @@ -import { faLink } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useDebounce } from "react-use"; import { Button } from "components/ui/Button"; +import { Icon } from "components/ui/Icon"; import { Tooltip } from "components/ui/Tooltip"; +import { buildAttemptLink } from "area/connection/utils/attemptLink"; import { copyToClipboard } from "core/utils/clipboard"; -import { buildAttemptLink } from "../attemptLinkUtils"; - interface Props { jobId: string | number; attemptId?: number; @@ -41,7 +39,7 @@ export const LinkToAttemptButton: React.FC = ({ jobId, attemptId }) => { onClick={onCopyLink} title={formatMessage({ id: "connection.copyLogLink" })} aria-label={formatMessage({ id: "connection.copyLogLink" })} - icon={} + icon={} /> } > diff --git a/airbyte-webapp/src/components/JobItem/types.ts b/airbyte-webapp/src/area/connection/types/jobs.ts similarity index 100% rename from airbyte-webapp/src/components/JobItem/types.ts rename to airbyte-webapp/src/area/connection/types/jobs.ts diff --git a/airbyte-webapp/src/components/JobItem/attemptLinkUtils.ts b/airbyte-webapp/src/area/connection/utils/attemptLink.ts similarity index 100% rename from airbyte-webapp/src/components/JobItem/attemptLinkUtils.ts rename to airbyte-webapp/src/area/connection/utils/attemptLink.ts diff --git a/airbyte-webapp/src/components/NewJobItem/isPartialSuccess.test.ts b/airbyte-webapp/src/area/connection/utils/jobs.test.ts similarity index 76% rename from airbyte-webapp/src/components/NewJobItem/isPartialSuccess.test.ts rename to airbyte-webapp/src/area/connection/utils/jobs.test.ts index 8063f148fc8..4cd69bbd936 100644 --- a/airbyte-webapp/src/components/NewJobItem/isPartialSuccess.test.ts +++ b/airbyte-webapp/src/area/connection/utils/jobs.test.ts @@ -2,11 +2,11 @@ import { mockAttempt } from "test-utils/mock-data/mockAttempt"; import { AttemptRead, JobStatus } from "core/request/AirbyteClient"; -import { isPartialSuccess } from "./isPartialSuccess"; +import { isJobPartialSuccess } from "./jobs"; -describe(`${isPartialSuccess.name}`, () => { +describe(`${isJobPartialSuccess.name}`, () => { it("should return false if attempts is undefined", () => { - expect(isPartialSuccess(undefined)).toBe(false); + expect(isJobPartialSuccess(undefined)).toBe(false); }); it("should return true if at least one attempt is a partial success", () => { @@ -25,7 +25,7 @@ describe(`${isPartialSuccess.name}`, () => { }, ]; - expect(isPartialSuccess(attempts)).toBe(true); + expect(isJobPartialSuccess(attempts)).toBe(true); }); it("should return false if no attempts are a partial success", () => { @@ -40,6 +40,6 @@ describe(`${isPartialSuccess.name}`, () => { }, ]; - expect(isPartialSuccess(attempts)).toBe(false); + expect(isJobPartialSuccess(attempts)).toBe(false); }); }); diff --git a/airbyte-webapp/src/area/connection/utils/jobs.ts b/airbyte-webapp/src/area/connection/utils/jobs.ts new file mode 100644 index 00000000000..d8f4241a851 --- /dev/null +++ b/airbyte-webapp/src/area/connection/utils/jobs.ts @@ -0,0 +1,25 @@ +import { AttemptRead, JobStatus, SynchronousJobRead } from "core/request/AirbyteClient"; + +import { JobWithAttempts } from "../types/jobs"; + +export const didJobSucceed = (job: SynchronousJobRead | JobWithAttempts): boolean => + "succeeded" in job ? job.succeeded : getJobStatus(job) !== "failed"; + +export const getJobStatus: (job: SynchronousJobRead | JobWithAttempts) => JobStatus = (job) => + "succeeded" in job ? (job.succeeded ? JobStatus.succeeded : JobStatus.failed) : job.job.status; + +export const getJobAttempts: (job: SynchronousJobRead | JobWithAttempts) => AttemptRead[] | undefined = (job) => + "attempts" in job ? job.attempts : undefined; + +export const getJobCreatedAt = (job: SynchronousJobRead | JobWithAttempts) => + (job as SynchronousJobRead).createdAt ?? (job as JobWithAttempts).job.createdAt; + +export const isJobPartialSuccess = (attempts?: AttemptRead[]) => { + if (!attempts) { + return false; + } + if (attempts.length > 0 && attempts[attempts.length - 1].status === JobStatus.failed) { + return attempts.some((attempt) => attempt.failureSummary && attempt.failureSummary.partialSuccess); + } + return false; +}; diff --git a/airbyte-webapp/src/area/connector/components/ArrayOfObjectsSection/ArrayOfObjectsSection.tsx b/airbyte-webapp/src/area/connector/components/ArrayOfObjectsSection/ArrayOfObjectsSection.tsx index ada3bda3de6..5755c1a97db 100644 --- a/airbyte-webapp/src/area/connector/components/ArrayOfObjectsSection/ArrayOfObjectsSection.tsx +++ b/airbyte-webapp/src/area/connector/components/ArrayOfObjectsSection/ArrayOfObjectsSection.tsx @@ -1,3 +1,4 @@ +import { JSONSchema7Type } from "json-schema"; import get from "lodash/get"; import React, { useState } from "react"; import { useFieldArray, useFormContext, useWatch } from "react-hook-form"; @@ -97,10 +98,34 @@ const stringify = (value: unknown): string => { const getItemName = (item: Record, properties: FormBlock[]): string => { return Object.keys(item) .sort() + .filter((key) => item[key] !== undefined && item[key] !== null && item[key] !== "") + .filter((key) => + // do not show empty objects + typeof item[key] === "object" && !Array.isArray(item[key]) + ? Object.keys(item[key] as Record).length > 0 + : true + ) .map((key) => { const property = properties.find(({ fieldKey }) => fieldKey === key); const name = property?.title ?? key; - return `${name}: ${stringify(item[key])}`; + const value = item[key]; + if (!property) { + return `${name}: ${stringify(value)}`; + } + if (property._type === "formItem") { + return `${name}: ${property.isSecret ? "*****" : stringify(value)}`; + } + if (property._type === "formGroup") { + return getItemName(value as Record, property.properties); + } + if (property._type === "formCondition") { + const selectionValue = (value as Record)[property.selectionKey] as JSONSchema7Type; + const selectedOption = property.conditions[property.selectionConstValues.indexOf(selectionValue)]; + return `${name}: ${selectedOption?.title ?? stringify(selectionValue)}`; + } + const arrayValue = value as Array>; + return `${name}: [${arrayValue.length > 0 ? "..." : ""}]`; }) + .filter(Boolean) .join(" | "); }; diff --git a/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx index b88e443d518..d05918d5dbe 100644 --- a/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx +++ b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx @@ -9,11 +9,10 @@ import { Heading } from "components/ui/Heading"; import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; +import { useDestinationDefinitionList, useSourceDefinitionList } from "core/api"; import { ConnectorDefinition } from "core/domain/connector"; import { isSourceDefinition } from "core/domain/connector/source"; import { useLocalStorage } from "core/utils/useLocalStorage"; -import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; -import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; import styles from "./SuggestedConnectors.module.scss"; diff --git a/airbyte-webapp/src/area/workspace/NoWorkspacesPermissionWarning.module.scss b/airbyte-webapp/src/area/workspace/NoWorkspacesPermissionWarning.module.scss new file mode 100644 index 00000000000..2962af240ac --- /dev/null +++ b/airbyte-webapp/src/area/workspace/NoWorkspacesPermissionWarning.module.scss @@ -0,0 +1,4 @@ +.cloudWorkspacesPage__illustration { + display: block; + margin: auto; +} diff --git a/airbyte-webapp/src/area/workspace/NoWorkspacesPermissionWarning.tsx b/airbyte-webapp/src/area/workspace/NoWorkspacesPermissionWarning.tsx new file mode 100644 index 00000000000..a534f404f79 --- /dev/null +++ b/airbyte-webapp/src/area/workspace/NoWorkspacesPermissionWarning.tsx @@ -0,0 +1,39 @@ +import { FormattedMessage } from "react-intl"; + +import { Box } from "components/ui/Box"; +import { FlexContainer } from "components/ui/Flex"; +import { ExternalLink } from "components/ui/Link"; +import { Text } from "components/ui/Text"; + +import { OrganizationRead } from "core/request/AirbyteClient"; + +import styles from "./NoWorkspacesPermissionWarning.module.scss"; +import OctaviaThinking from "./octavia-thinking-no-gears.svg?react"; + +export const NoWorkspacePermissionsContent: React.FC<{ organizations: OrganizationRead[] }> = ({ organizations }) => { + return ( + + + +
+ + + + + + + ( + {lnk} + ), + }} + /> + +
+ + + ); +}; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/octavia-thinking-no-gears.svg b/airbyte-webapp/src/area/workspace/octavia-thinking-no-gears.svg similarity index 100% rename from airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/octavia-thinking-no-gears.svg rename to airbyte-webapp/src/area/workspace/octavia-thinking-no-gears.svg diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx deleted file mode 100644 index dafd2e8da89..00000000000 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx +++ /dev/null @@ -1,98 +0,0 @@ -import React from "react"; -import { FormattedMessage } from "react-intl"; - -import { Box } from "components/ui/Box"; -import { FlexContainer } from "components/ui/Flex"; -import { Modal, ModalProps } from "components/ui/Modal"; - -import { ConnectionFormMode } from "hooks/services/ConnectionForm/ConnectionFormService"; - -import styles from "./ArrayOfObjectsEditor.module.scss"; -import { EditorHeader } from "./components/EditorHeader"; -import { EditorRow } from "./components/EditorRow"; - -interface ItemBase { - name?: string; - description?: string; -} - -export interface ArrayOfObjectsEditorProps { - items: T[]; - editableItemIndex?: number | string | null; - mainTitle?: React.ReactNode; - addButtonText?: React.ReactNode; - renderItemName?: (item: T, index: number) => React.ReactNode | undefined; - renderItemDescription?: (item: T, index: number) => React.ReactNode | undefined; - renderItemEditorForm: (item?: T) => React.ReactNode; - onStartEdit: (n: number) => void; - onRemove: (index: number) => void; - onCancel?: () => void; - mode?: ConnectionFormMode; - disabled?: boolean; - editModalSize?: ModalProps["size"]; -} - -export const ArrayOfObjectsEditor = ({ - onStartEdit, - onRemove, - onCancel, - renderItemName = (item) => item.name, - renderItemDescription = (item) => item.description, - renderItemEditorForm, - items, - editableItemIndex, - mainTitle, - addButtonText, - mode, - disabled, - editModalSize, -}: ArrayOfObjectsEditorProps): JSX.Element => { - const onAddItem = React.useCallback(() => onStartEdit(items.length), [onStartEdit, items]); - const isEditable = editableItemIndex !== null && editableItemIndex !== undefined; - - const renderEditModal = () => { - const item = typeof editableItemIndex === "number" ? items[editableItemIndex] : undefined; - - return ( - } - size={editModalSize} - testId="arrayOfObjects-editModal" - onClose={onCancel} - > - {renderItemEditorForm(item)} - - ); - }; - - return ( - <> - - - {items.length ? ( - - {items.map((item, index) => ( - - ))} - - ) : null} - - {mode !== "readonly" && isEditable && renderEditModal()} - - ); -}; diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsHookFormEditor.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsHookFormEditor.tsx index c6834cc6267..1f5a55d7dfc 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsHookFormEditor.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsHookFormEditor.tsx @@ -22,7 +22,6 @@ export interface ArrayOfObjectsHookFormEditorProps { /** * The component is used to render a list of react-hook-form FieldArray items with the ability to add, edit and remove items. * It's a react-hook-form version of the ArrayOfObjectsEditor component and will replace it in the future. - * @see ArrayOfObjectsEditor * @param fields * @param mainTitle * @param addButtonText diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx index d8b44d1689c..3c44d60ecdd 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx @@ -1,2 +1 @@ -export { ArrayOfObjectsEditor } from "./ArrayOfObjectsEditor"; export { ArrayOfObjectsHookFormEditor } from "./ArrayOfObjectsHookFormEditor"; diff --git a/airbyte-webapp/src/components/ConnectorBuilderProjectTable/ConnectorBuilderProjectTable.tsx b/airbyte-webapp/src/components/ConnectorBuilderProjectTable/ConnectorBuilderProjectTable.tsx index 262b3576a17..75562b1e2d9 100644 --- a/airbyte-webapp/src/components/ConnectorBuilderProjectTable/ConnectorBuilderProjectTable.tsx +++ b/airbyte-webapp/src/components/ConnectorBuilderProjectTable/ConnectorBuilderProjectTable.tsx @@ -1,5 +1,3 @@ -import { faTrashCan, faPenToSquare, faCaretDown } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { createColumnHelper } from "@tanstack/react-table"; import classNames from "classnames"; import { useMemo, useState } from "react"; @@ -8,6 +6,7 @@ import { useNavigate } from "react-router-dom"; import { Button } from "components/ui/Button"; import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { Modal, ModalBody } from "components/ui/Modal"; import { Spinner } from "components/ui/Spinner"; import { Table } from "components/ui/Table"; @@ -20,8 +19,7 @@ import { useDeleteBuilderProject, useListBuilderProjectVersions, } from "core/api"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useNotificationService } from "hooks/services/Notification"; import { getEditPath } from "pages/connectorBuilder/ConnectorBuilderRoutes"; @@ -160,7 +158,7 @@ const VersionChanger = ({ project }: { project: BuilderProject }) => { onClick={() => { setChangeInProgress(true); }} - icon={} + icon={} iconPosition="right" data-testid={`version-changer-${project.name}`} > @@ -212,7 +210,7 @@ export const ConnectorBuilderProjectTable = ({ + + ); +}; diff --git a/airbyte-webapp/src/components/DeployPreviewMessage/index.tsx b/airbyte-webapp/src/components/DeployPreviewMessage/index.tsx new file mode 100644 index 00000000000..19b52e2ca30 --- /dev/null +++ b/airbyte-webapp/src/components/DeployPreviewMessage/index.tsx @@ -0,0 +1 @@ +export * from "./DeployPreviewMessage"; diff --git a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx index 073ffd99ee9..b49246b67ef 100644 --- a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx +++ b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx @@ -21,9 +21,10 @@ interface ConnectionTableProps { data: ConnectionTableDataItem[]; entity: "source" | "destination" | "connection"; onClickRow?: (data: ConnectionTableDataItem) => void; + variant?: React.ComponentProps["variant"]; } -const ConnectionTable: React.FC = ({ data, entity, onClickRow }) => { +const ConnectionTable: React.FC = ({ data, entity, onClickRow, variant }) => { const allowAutoDetectSchema = useFeature(FeatureItem.AllowAutoDetectSchema); const streamCentricUIEnabled = false; @@ -64,6 +65,7 @@ const ConnectionTable: React.FC = ({ data, entity, onClick cell: (props) => ( = ({ data, entity, onClick cell: (props) => ( @@ -139,6 +142,7 @@ const ConnectionTable: React.FC = ({ data, entity, onClick return ( = ({ con }, [connectEntities]); return statusIconProps ? ( - + ) : null; }; diff --git a/airbyte-webapp/src/components/EntityTable/components/ChangesStatusIcon.tsx b/airbyte-webapp/src/components/EntityTable/components/ChangesStatusIcon.tsx index 7379646d0f8..26dfc6b1f34 100644 --- a/airbyte-webapp/src/components/EntityTable/components/ChangesStatusIcon.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/ChangesStatusIcon.tsx @@ -1,9 +1,8 @@ -import { faExclamationCircle, faInfoCircle } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; import React from "react"; import { FormattedMessage } from "react-intl"; +import { Icon } from "components/ui/Icon"; import { Tooltip } from "components/ui/Tooltip"; import { SchemaChange } from "core/request/AirbyteClient"; @@ -28,10 +27,10 @@ export const ChangesStatusIcon: React.FC = ({ schemaChan placement="left" containerClassName={styles.tooltipContainer} control={ - } diff --git a/airbyte-webapp/src/components/EntityTable/components/ConnectionSettingsCell.tsx b/airbyte-webapp/src/components/EntityTable/components/ConnectionSettingsCell.tsx index 17756c7dbc1..7de3a818713 100644 --- a/airbyte-webapp/src/components/EntityTable/components/ConnectionSettingsCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/ConnectionSettingsCell.tsx @@ -1,12 +1,10 @@ -import { faCog } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; +import { Icon } from "components/ui/Icon"; import { Link } from "components/ui/Link"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; -import { ConnectionRoutePaths } from "pages/routePaths"; -import { RoutePaths } from "pages/routePaths"; +import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; import styles from "./ConnectionSettingsCell.module.scss"; @@ -26,7 +24,7 @@ const ConnectorCell: React.FC = ({ id }) => { return ( ); diff --git a/airbyte-webapp/src/components/EntityTable/components/ConnectorNameCell.tsx b/airbyte-webapp/src/components/EntityTable/components/ConnectorNameCell.tsx index 85e5e8027de..f5d4b8f1f93 100644 --- a/airbyte-webapp/src/components/EntityTable/components/ConnectorNameCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/ConnectorNameCell.tsx @@ -2,22 +2,34 @@ import React from "react"; import { ConnectorIcon } from "components/common/ConnectorIcon"; import { FlexContainer } from "components/ui/Flex"; +import { Tooltip } from "components/ui/Tooltip"; import styles from "./ConnectorNameCell.module.scss"; import { EntityNameCell } from "./EntityNameCell"; interface ConnectorNameCellProps { enabled: boolean; + /** + * connector name defined by user + * @example: "My PokeAPI", "Postgres - Source", "Shopify_123" + */ value: string; + /** + * the actual name of connector + * @example: "PokeAPI", "Postgres", "Shopify" + */ + actualName?: string; icon: string | undefined; hideIcon?: boolean; } -export const ConnectorNameCell: React.FC = ({ value, enabled, icon, hideIcon }) => { - return ( - - {!hideIcon && } - - - ); -}; +export const ConnectorNameCell: React.FC = ({ value, actualName, enabled, icon, hideIcon }) => ( + + {actualName ? ( + }>{actualName} + ) : ( + !hideIcon && + )} + + +); diff --git a/airbyte-webapp/src/components/EntityTable/utils.tsx b/airbyte-webapp/src/components/EntityTable/utils.tsx index 150c849201d..8d5a72abb9d 100644 --- a/airbyte-webapp/src/components/EntityTable/utils.tsx +++ b/airbyte-webapp/src/components/EntityTable/utils.tsx @@ -80,14 +80,8 @@ export const getConnectionTableData = ( return connections.map((connection) => ({ connectionId: connection.connectionId, name: connection.name, - entityName: - type === "connection" - ? `${connection.source?.sourceName} - ${connection.source?.name}` - : connection[connectType]?.name || "", - connectorName: - type === "connection" - ? `${connection.destination?.destinationName} - ${connection.destination?.name}` - : getConnectorTypeName(connection[connectType]), + entityName: type === "connection" ? connection.source?.name : connection[connectType]?.name || "", + connectorName: type === "connection" ? connection.destination?.name : getConnectorTypeName(connection[connectType]), lastSync: connection.latestSyncJobCreatedAt, enabled: connection.status === ConnectionStatus.active, schemaChange: connection.schemaChange, diff --git a/airbyte-webapp/src/components/JobFailure/JobFailure.tsx b/airbyte-webapp/src/components/JobFailure/JobFailure.tsx index 4c67a987b24..9ccb3151457 100644 --- a/airbyte-webapp/src/components/JobFailure/JobFailure.tsx +++ b/airbyte-webapp/src/components/JobFailure/JobFailure.tsx @@ -1,11 +1,10 @@ -import { faAngleDown, faAngleRight, faFileDownload } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import { FormattedMessage, useIntl } from "react-intl"; import { useToggle } from "react-use"; import Logs from "components/Logs"; import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { Message } from "components/ui/Message"; import { Text } from "components/ui/Text"; @@ -47,7 +46,7 @@ const DisclosureHeader = ({ ); }; diff --git a/airbyte-webapp/src/components/JobItem/JobItem.module.scss b/airbyte-webapp/src/components/JobItem/JobItem.module.scss deleted file mode 100644 index 265c0d9bfc7..00000000000 --- a/airbyte-webapp/src/components/JobItem/JobItem.module.scss +++ /dev/null @@ -1,21 +0,0 @@ -@use "scss/colors"; -@use "scss/variables"; - -.jobItem { - &:not(:last-child) { - border-bottom: variables.$border-thin solid colors.$grey-100; - } - - &:hover { - background-color: colors.$grey-50; - } - - &__loading { - display: flex; - align-items: center; - justify-content: center; - background: colors.$foreground; - padding: 6px 0; - min-height: 58px; - } -} diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx deleted file mode 100644 index 33f80110de3..00000000000 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ /dev/null @@ -1,66 +0,0 @@ -import React, { Suspense, useCallback, useRef, useState } from "react"; - -import { Spinner } from "components/ui/Spinner"; - -import { SynchronousJobRead } from "core/request/AirbyteClient"; - -import { useAttemptLink } from "./attemptLinkUtils"; -import ContentWrapper from "./components/ContentWrapper"; -import { JobSummary } from "./components/JobSummary"; -import styles from "./JobItem.module.scss"; -import { JobWithAttempts } from "./types"; -import { didJobSucceed, getJobAttempts, getJobId } from "./utils"; - -const ErrorDetails = React.lazy(() => import("./components/ErrorDetails")); -const JobLogs = React.lazy(() => import("./components/JobLogs")); - -interface JobItemProps { - job: SynchronousJobRead | JobWithAttempts; -} - -export const JobItem: React.FC = ({ job }) => { - const { jobId: linkedJobId } = useAttemptLink(); - const alreadyScrolled = useRef(false); - const [isOpen, setIsOpen] = useState(() => linkedJobId === String(getJobId(job))); - const scrollAnchor = useRef(null); - - const didSucceed = didJobSucceed(job); - - const onExpand = () => { - setIsOpen((prevIsOpen) => !prevIsOpen); - }; - - const onDetailsToggled = useCallback(() => { - if (alreadyScrolled.current || linkedJobId !== String(getJobId(job))) { - return; - } - scrollAnchor.current?.scrollIntoView({ - block: "start", - }); - alreadyScrolled.current = true; - }, [job, linkedJobId]); - - return ( -
- - -
- - -
- } - > - {isOpen && ( - <> - - - - )} - -
- - - ); -}; diff --git a/airbyte-webapp/src/components/JobItem/components/ContentWrapper.module.scss b/airbyte-webapp/src/components/JobItem/components/ContentWrapper.module.scss deleted file mode 100644 index 475385588f6..00000000000 --- a/airbyte-webapp/src/components/JobItem/components/ContentWrapper.module.scss +++ /dev/null @@ -1,3 +0,0 @@ -.container { - overflow: hidden; -} diff --git a/airbyte-webapp/src/components/JobItem/components/ContentWrapper.tsx b/airbyte-webapp/src/components/JobItem/components/ContentWrapper.tsx deleted file mode 100644 index 6a43ac76ebc..00000000000 --- a/airbyte-webapp/src/components/JobItem/components/ContentWrapper.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import { motion } from "framer-motion"; -import React from "react"; - -import styles from "./ContentWrapper.module.scss"; - -interface IProps { - children?: React.ReactNode; - isOpen?: boolean; - onToggled?: () => void; -} - -const ContentWrapper: React.FC> = ({ children, isOpen, onToggled }) => { - return ( - - {children} - - ); -}; - -export default ContentWrapper; diff --git a/airbyte-webapp/src/components/JobItem/components/DebugInfoDetailsModal.module.scss b/airbyte-webapp/src/components/JobItem/components/DebugInfoDetailsModal.module.scss deleted file mode 100644 index d1566a69a9f..00000000000 --- a/airbyte-webapp/src/components/JobItem/components/DebugInfoDetailsModal.module.scss +++ /dev/null @@ -1,3 +0,0 @@ -.buttonWithMargin { - margin-right: 9px; -} diff --git a/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx b/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx deleted file mode 100644 index 45d288da779..00000000000 --- a/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx +++ /dev/null @@ -1,40 +0,0 @@ -import { faFileDownload } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import React from "react"; -import { useIntl } from "react-intl"; - -import { Button } from "components/ui/Button"; - -import { useCurrentWorkspace } from "core/api"; -import { JobDebugInfoRead } from "core/request/AirbyteClient"; -import { FILE_TYPE_DOWNLOAD, downloadFile, fileizeString } from "core/utils/file"; - -interface DownloadButtonProps { - jobDebugInfo: JobDebugInfoRead; - fileName: string; -} - -const DownloadButton: React.FC = ({ jobDebugInfo, fileName }) => { - const { formatMessage } = useIntl(); - const { name } = useCurrentWorkspace(); - - const downloadFileWithLogs = () => { - const file = new Blob([jobDebugInfo.attempts.flatMap((info) => info.logs.logLines).join("\n")], { - type: FILE_TYPE_DOWNLOAD, - }); - downloadFile(file, fileizeString(`${name}-${fileName}.txt`)); - }; - - return ( - - - - - - - - {/* tabIndex can be -1 because the main button is still focusable. This one is just for convenience. */} - - - - ); -}; diff --git a/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx b/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx deleted file mode 100644 index 2dd883cb790..00000000000 --- a/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx +++ /dev/null @@ -1,40 +0,0 @@ -import React from "react"; - -import Logs from "components/Logs"; -import { Box } from "components/ui/Box"; -import { FlexContainer, FlexItem } from "components/ui/Flex"; -import { Text } from "components/ui/Text"; - -import { AttemptDetails } from "./AttemptDetails"; -import DownloadButton from "./DownloadButton"; -import { LinkToAttemptButton } from "./LinkToAttemptButton"; -import { AttemptRead, JobDebugInfoRead } from "../../../core/request/AirbyteClient"; - -export const LogsDetails: React.FC<{ - jobId: string; - path: string; - currentAttempt?: AttemptRead; - jobDebugInfo?: JobDebugInfoRead; - showAttemptStats: boolean; - logs?: string[]; -}> = ({ path, jobId, currentAttempt, jobDebugInfo, showAttemptStats, logs }) => ( - <> - {currentAttempt && showAttemptStats && ( - - - - )} - - - - - {path} - - - - {jobDebugInfo && } - - - - -); diff --git a/airbyte-webapp/src/components/JobItem/index.ts b/airbyte-webapp/src/components/JobItem/index.ts deleted file mode 100644 index 79acb143ceb..00000000000 --- a/airbyte-webapp/src/components/JobItem/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./JobItem"; diff --git a/airbyte-webapp/src/components/JobItem/utils.ts b/airbyte-webapp/src/components/JobItem/utils.ts deleted file mode 100644 index 7ae1c3588e3..00000000000 --- a/airbyte-webapp/src/components/JobItem/utils.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { AttemptRead, FailureReason, FailureType, JobStatus, SynchronousJobRead } from "core/request/AirbyteClient"; - -import { JobWithAttempts } from "./types"; - -export const getFailureFromAttempt = (attempt: AttemptRead): FailureReason | undefined => - attempt.failureSummary?.failures[0]; - -export const isCancelledAttempt = (attempt: AttemptRead): boolean => - attempt.failureSummary?.failures.some(({ failureType }) => failureType === FailureType.manual_cancellation) ?? false; - -export const didJobSucceed = (job: SynchronousJobRead | JobWithAttempts): boolean => - "succeeded" in job ? job.succeeded : getJobStatus(job) !== "failed"; - -export const getJobStatus: (job: SynchronousJobRead | JobWithAttempts) => JobStatus = (job) => - "succeeded" in job ? (job.succeeded ? JobStatus.succeeded : JobStatus.failed) : job.job.status; - -export const getJobAttempts: (job: SynchronousJobRead | JobWithAttempts) => AttemptRead[] | undefined = (job) => - "attempts" in job ? job.attempts : undefined; - -export const getJobId = (job: SynchronousJobRead | JobWithAttempts): string => - "id" in job ? job.id : String(job.job.id); diff --git a/airbyte-webapp/src/components/NewJobItem/JobStatusIcon.tsx b/airbyte-webapp/src/components/NewJobItem/JobStatusIcon.tsx deleted file mode 100644 index ae6b2fccd5c..00000000000 --- a/airbyte-webapp/src/components/NewJobItem/JobStatusIcon.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import { JobWithAttempts } from "components/JobItem/types"; -import { didJobSucceed, getJobStatus } from "components/JobItem/utils"; -import { StatusIcon } from "components/ui/StatusIcon"; - -import { AttemptInfoRead, JobStatus, AttemptStatus } from "core/request/AirbyteClient"; - -import { isPartialSuccess } from "./isPartialSuccess"; - -interface JobStatusIconProps { - job: JobWithAttempts; -} - -export const JobStatusIcon: React.FC = ({ job }) => { - const didSucceed = didJobSucceed(job); - const jobStatus = getJobStatus(job); - const jobIsPartialSuccess = isPartialSuccess(job.attempts); - - if (jobIsPartialSuccess) { - return ; - } else if (!didSucceed) { - return ; - } else if (jobStatus === JobStatus.cancelled) { - return ; - } else if (jobStatus === JobStatus.running || jobStatus === JobStatus.incomplete) { - return ; - } else if (jobStatus === JobStatus.succeeded) { - return ; - } - return null; -}; - -interface AttemptStatusIconProps { - attempt: AttemptInfoRead; -} - -export const AttemptStatusIcon: React.FC = ({ attempt }) => { - if (attempt.attempt.status === AttemptStatus.failed) { - return ; - } else if (attempt.attempt.status === AttemptStatus.running) { - return ; - } else if (attempt.attempt.status === AttemptStatus.succeeded) { - return ; - } - return null; -}; diff --git a/airbyte-webapp/src/components/NewJobItem/index.ts b/airbyte-webapp/src/components/NewJobItem/index.ts deleted file mode 100644 index d632593d09b..00000000000 --- a/airbyte-webapp/src/components/NewJobItem/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { NewJobItem } from "./NewJobItem"; diff --git a/airbyte-webapp/src/components/NewJobItem/isPartialSuccess.ts b/airbyte-webapp/src/components/NewJobItem/isPartialSuccess.ts deleted file mode 100644 index d1e03ad3215..00000000000 --- a/airbyte-webapp/src/components/NewJobItem/isPartialSuccess.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { AttemptRead, JobStatus } from "core/request/AirbyteClient"; - -export const isPartialSuccess = (attempts?: AttemptRead[]) => { - if (!attempts) { - return false; - } - if (attempts.length > 0 && attempts[attempts.length - 1].status === JobStatus.failed) { - return attempts.some((attempt) => attempt.failureSummary && attempt.failureSummary.partialSuccess); - } - return false; -}; diff --git a/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.module.scss b/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.module.scss deleted file mode 100644 index 4a2deba14de..00000000000 --- a/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.module.scss +++ /dev/null @@ -1,3 +0,0 @@ -.flag { - width: 16px; -} diff --git a/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.tsx b/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.tsx deleted file mode 100644 index e7e882e1608..00000000000 --- a/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.tsx +++ /dev/null @@ -1,52 +0,0 @@ -import * as Flags from "country-flag-icons/react/3x2"; -import { useIntl } from "react-intl"; - -import { ListBox } from "components/ui/ListBox"; - -import { Geography } from "core/request/AirbyteClient"; - -import styles from "./DataGeographyDropdown.module.scss"; - -interface DataGeographyDropdownProps { - geographies: Geography[]; - isDisabled?: boolean; - onChange: (value: Geography) => void; - value: Geography; -} - -/** - * @deprecated it's not form related component and will be removed in the future, use DataResidencyDropdown instead - * @param geographies - * @param isDisabled - * @param onChange - * @param value - * @constructor - */ -export const DataGeographyDropdown: React.FC = ({ - geographies, - isDisabled = false, - onChange, - value, -}) => { - const { formatMessage } = useIntl(); - - return ( - { - const Flag = - geography === "auto" ? Flags.US : Flags[geography.toUpperCase() as Uppercase>]; - return { - label: formatMessage({ - id: `connection.geography.${geography}`, - defaultMessage: geography.toUpperCase(), - }), - value: geography, - icon: , - }; - })} - onSelect={onChange} - selectedValue={value} - isDisabled={isDisabled} - /> - ); -}; diff --git a/airbyte-webapp/src/components/common/DataGeographyDropdown/index.ts b/airbyte-webapp/src/components/common/DataGeographyDropdown/index.ts deleted file mode 100644 index 88c96d65a29..00000000000 --- a/airbyte-webapp/src/components/common/DataGeographyDropdown/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { DataGeographyDropdown } from "./DataGeographyDropdown"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordionHeader.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordionHeader.tsx index 7b14422096e..59e2c7ab204 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordionHeader.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordionHeader.tsx @@ -1,9 +1,7 @@ -import { faAngleDown, faAngleRight } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; import { useIntl } from "react-intl"; -import { ModificationIcon } from "components/icons/ModificationIcon"; +import { Icon } from "components/ui/Icon"; import { StreamDescriptor } from "core/request/AirbyteClient"; @@ -34,9 +32,9 @@ export const DiffAccordionHeader: React.FC = ({ return ( <> - +
- +
{namespace}
diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.module.scss b/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.module.scss index cdb7597d631..acd5415ed7f 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.module.scss +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.module.scss @@ -66,6 +66,7 @@ tr:last-child .content { .field { display: flex; + align-items: center; } .fieldName { diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.tsx index 0ffef1b7f59..059591e620d 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.tsx @@ -1,10 +1,7 @@ -import { faMinus, faPlus, faExclamationCircle } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; import { FormattedMessage } from "react-intl"; -import { ArrowRightIcon } from "components/icons/ArrowRightIcon"; -import { ModificationIcon } from "components/icons/ModificationIcon"; +import { Icon } from "components/ui/Icon"; import { Tooltip } from "components/ui/Tooltip"; import { FieldTransform } from "core/request/AirbyteClient"; @@ -50,12 +47,12 @@ export const FieldRow: React.FC = ({ transform }) => {
diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/StreamRow.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/StreamRow.tsx index f4b9b1d503b..482ff97d533 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/StreamRow.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/StreamRow.tsx @@ -1,8 +1,6 @@ -import { faMinus, faPlus } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; -import { ModificationIcon } from "components/icons/ModificationIcon"; +import { Icon } from "components/ui/Icon"; import { StreamTransform } from "core/request/AirbyteClient"; @@ -42,11 +40,11 @@ export const StreamRow: React.FC = ({ streamTransform, syncMode,
{diffVerb === "new" ? ( - + ) : diffVerb === "removed" ? ( - + ) : ( - + )}
diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationFormPreview.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationFormPreview.tsx deleted file mode 100644 index c550e5317a4..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationFormPreview.tsx +++ /dev/null @@ -1,129 +0,0 @@ -import { useFormikContext } from "formik"; -import React from "react"; -import { FormattedMessage } from "react-intl"; - -import { FlexContainer } from "components/ui/Flex"; -import { Text } from "components/ui/Text"; - -import { ConnectionScheduleType, NamespaceDefinitionType } from "core/request/AirbyteClient"; -import { FeatureItem, useFeature } from "core/services/features"; -import { useExperiment } from "hooks/services/Experiment"; - -import styles from "./ConnectionConfigurationFormPreview.module.scss"; -import { FormikConnectionFormValues } from "./formConfig"; -import { namespaceDefinitionOptions } from "./types"; - -const Frequency: React.FC = () => { - const { - values: { scheduleType, scheduleData }, - } = useFormikContext(); - - return ( -
- - : - - - {scheduleType === ConnectionScheduleType.manual && } - {scheduleType === ConnectionScheduleType.cron && ( - <> - - {scheduleData?.cron?.cronExpression}{" "} - {scheduleData?.cron?.cronTimeZone} - - )} - {scheduleType === ConnectionScheduleType.basic && ( - - )} - -
- ); -}; - -const DestinationNamespace: React.FC = () => { - const { - values: { namespaceDefinition, namespaceFormat }, - } = useFormikContext(); - - return ( -
- - : - - - - {namespaceDefinitionOptions[namespaceDefinition as NamespaceDefinitionType] === - namespaceDefinitionOptions.customformat && ( - <> - {" - "} - {namespaceFormat} - - )} - -
- ); -}; - -const DestinationPrefix: React.FC = () => { - const { - values: { prefix }, - } = useFormikContext(); - - return ( -
- - : - - - {prefix === "" ? ( - - ) : ( - prefix - )} - -
- ); -}; - -const NonBreakingChanges: React.FC<{ - allowAutoDetectSchema: boolean; -}> = ({ allowAutoDetectSchema }) => { - const { - values: { nonBreakingChangesPreference }, - } = useFormikContext(); - const autoPropagationEnabled = useExperiment("autopropagation.enabled", true); - const autoPropagationPrefix = autoPropagationEnabled ? "autopropagation." : ""; - const labelKey = autoPropagationEnabled - ? "connectionForm.nonBreakingChangesPreference.autopropagation.label" - : "connectionForm.nonBreakingChangesPreference.label"; - - return allowAutoDetectSchema ? ( -
- - : - - - - -
- ) : null; -}; - -export const ConnectionConfigurationFormPreview: React.FC = () => { - const allowAutoDetectSchema = useFeature(FeatureItem.AllowAutoDetectSchema); - - return ( - - - - - - - ); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationHookFormCard.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationHookFormCard.tsx index 3d4fd40e1e4..3869e535873 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationHookFormCard.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationHookFormCard.tsx @@ -2,7 +2,7 @@ import { FormattedMessage } from "react-intl"; import { CollapsibleCard } from "components/ui/CollapsibleCard"; -import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; +import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { ConnectionConfigurationHookFormPreview } from "./ConnectionConfigurationHookFormPreview"; import { DestinationStreamPrefixNameHookForm } from "./DestinationStreamPrefixNameHookForm"; @@ -15,7 +15,7 @@ import { ScheduleHookFormField } from "./ScheduleHookFormField/ScheduleHookFormF * this component is used in create and update connection cases */ export const ConnectionConfigurationHookFormCard = () => { - const { mode } = useConnectionHookFormService(); + const { mode } = useConnectionFormService(); const isEditMode = mode === "edit"; return ( diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.module.scss b/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.module.scss deleted file mode 100644 index c92ccade2fe..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.module.scss +++ /dev/null @@ -1,8 +0,0 @@ -@use "scss/variables"; - -.tryArrow { - margin: 0 variables.$spacing-md -1px 0; - - // used to control svg size - font-size: variables.$font-size-lg; -} diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.tsx deleted file mode 100644 index 87dd90e2f84..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.tsx +++ /dev/null @@ -1,91 +0,0 @@ -import { faSyncAlt } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { Field } from "formik"; -import React from "react"; -import { FormattedMessage } from "react-intl"; -import { useEffectOnce } from "react-use"; - -import { FormChangeTracker } from "components/common/FormChangeTracker"; -import { Button } from "components/ui/Button"; -import { Card } from "components/ui/Card"; -import { CollapsibleCard } from "components/ui/CollapsibleCard"; -import { FlexContainer } from "components/ui/Flex"; - -import { FeatureItem, useFeature } from "core/services/features"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; - -import { ConnectionConfigurationFormPreview } from "./ConnectionConfigurationFormPreview"; -import styles from "./ConnectionFormFields.module.scss"; -import { DestinationStreamPrefixName } from "./DestinationStreamPrefixName"; -import { NamespaceDefinitionField } from "./NamespaceDefinitionField"; -import { NonBreakingChangesPreferenceField } from "./NonBreakingChangesPreferenceField"; -import { useRefreshSourceSchemaWithConfirmationOnDirty } from "./refreshSourceSchemaWithConfirmationOnDirty"; -import { ScheduleField } from "./ScheduleField"; -import { SyncCatalogField } from "./SyncCatalogField"; - -interface ConnectionFormFieldsProps { - isSubmitting: boolean; - dirty: boolean; - validateForm?: () => void; -} - -export const ConnectionFormFields: React.FC = ({ isSubmitting, dirty, validateForm }) => { - const allowAutoDetectSchema = useFeature(FeatureItem.AllowAutoDetectSchema); - - const { mode, formId } = useConnectionFormService(); - - const refreshSchema = useRefreshSourceSchemaWithConfirmationOnDirty(dirty); - - // If the source doesn't select any streams by default, the initial untouched state - // won't validate that at least one is selected. In this case, a user could submit the form - // without selecting any streams, which would trigger an error and cause a lousy UX. - useEffectOnce(() => { - validateForm?.(); - }); - - const isEditMode = mode === "edit"; - - return ( - <> - {/* FormChangeTracker is here as it has access to everything it needs without being repeated */} - - - } - collapsible={isEditMode} - defaultCollapsedState={isEditMode} - collapsedPreviewInfo={} - testId="configuration" - > - - - - - {allowAutoDetectSchema && ( - - )} - - - - - - - - } - /> - - - - ); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/CreateControls.module.scss b/airbyte-webapp/src/components/connection/ConnectionForm/CreateControls.module.scss deleted file mode 100644 index 606be88809f..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/CreateControls.module.scss +++ /dev/null @@ -1,11 +0,0 @@ -@use "scss/variables"; -@use "scss/colors"; - -.container { - margin-top: variables.$spacing-md; -} - -.errorText { - align-self: center; - color: colors.$red; -} diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/CreateControls.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/CreateControls.tsx deleted file mode 100644 index b4291363ff5..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/CreateControls.tsx +++ /dev/null @@ -1,31 +0,0 @@ -import React from "react"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/ui/Button"; -import { FlexContainer } from "components/ui/Flex"; - -import styles from "./CreateControls.module.scss"; - -interface CreateControlsProps { - isSubmitting: boolean; - isValid: boolean; - errorMessage?: React.ReactNode; -} - -/** - * @deprecated will be removed during clean up - https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - * use CreateControlsHookForm.tsx instead - * @see CreateControlsHookForm - */ -export const CreateControls: React.FC = ({ isSubmitting, errorMessage, isValid }) => { - return ( - -
{errorMessage}
-
- -
- - ); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/CreateControlsHookForm.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/CreateControlsHookForm.tsx index 5074f9fde92..d1386794c31 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/CreateControlsHookForm.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/CreateControlsHookForm.tsx @@ -8,7 +8,7 @@ import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; -import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; +import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { HookFormConnectionFormValues } from "./hookFormConfig"; @@ -20,7 +20,7 @@ import { HookFormConnectionFormValues } from "./hookFormConfig"; export const CreateControlsHookForm: React.FC = () => { const { isSubmitting, isValid, errors } = useFormState(); const { trigger } = useFormContext(); - const { getErrorMessage } = useConnectionHookFormService(); + const { getErrorMessage } = useConnectionFormService(); const errorMessage = getErrorMessage(isValid, errors); // If the source doesn't select any streams by default, the initial untouched state diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/DestinationStreamPrefixName.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/DestinationStreamPrefixName.tsx deleted file mode 100644 index 76a535c0ef1..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/DestinationStreamPrefixName.tsx +++ /dev/null @@ -1,91 +0,0 @@ -import { Field, FieldProps, useFormikContext } from "formik"; -import { useCallback } from "react"; -import { FormattedMessage, useIntl } from "react-intl"; - -import { ControlLabels } from "components/LabeledControl"; -import { Button } from "components/ui/Button"; -import { FlexContainer } from "components/ui/Flex"; -import { Text } from "components/ui/Text"; -import { TextInputContainer } from "components/ui/TextInputContainer"; - -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useModalService } from "hooks/services/Modal"; - -import { FormikConnectionFormValues } from "./formConfig"; -import { FormFieldLayout } from "./FormFieldLayout"; -import { - DestinationStreamNamesModal, - DestinationStreamNamesFormValues, - StreamNameDefinitionValueType, -} from "../DestinationStreamNamesModal"; - -export const DestinationStreamPrefixName = () => { - const { mode } = useConnectionFormService(); - const { formatMessage } = useIntl(); - const { openModal, closeModal } = useModalService(); - const formikProps = useFormikContext(); - - const destinationStreamNamesHookFormChange = useCallback( - (value: DestinationStreamNamesFormValues) => { - formikProps.setFieldValue( - "prefix", - value.streamNameDefinition === StreamNameDefinitionValueType.Prefix ? value.prefix : "" - ); - }, - [formikProps] - ); - - const openDestinationStreamNamesModal = useCallback( - () => - openModal({ - size: "sm", - title: , - content: () => ( - - ), - }), - [closeModal, destinationStreamNamesHookFormChange, formikProps.values.prefix, openModal] - ); - return ( - - {({ field }: FieldProps) => ( - - - - - - {!field.value - ? formatMessage({ id: "connectionForm.modal.destinationStreamNames.radioButton.mirror" }) - : field.value} - - - - - - )} - - ); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss deleted file mode 100644 index 91e9494cd0f..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.module.scss +++ /dev/null @@ -1,10 +0,0 @@ -@use "scss/variables"; - -.container { - margin-top: variables.$spacing-md; - transition: opacity variables.$transition-out; - - &.hidden { - opacity: 0; - } -} diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx deleted file mode 100644 index 852e98823d7..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/EditControls.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import classNames from "classnames"; -import React from "react"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/ui/Button"; -import { FlexContainer } from "components/ui/Flex"; - -import styles from "./EditControls.module.scss"; -import { ResponseMessage } from "./ResponseMessage"; - -interface EditControlProps { - isSubmitting: boolean; - dirty: boolean; - submitDisabled?: boolean; - resetForm: () => void; - successMessage?: React.ReactNode; - errorMessage?: React.ReactNode; - enableControls?: boolean; - hidden?: boolean; -} - -const EditControls: React.FC = ({ - isSubmitting, - dirty, - submitDisabled, - resetForm, - successMessage, - errorMessage, - enableControls, - hidden, -}) => { - const isButtonDisabled = hidden || isSubmitting || (!dirty && !enableControls); - return ( - - ); -}; - -export default EditControls; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/EditControlsHookForm.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/EditControlsHookForm.tsx index a6d3ec18460..88bb374e770 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/EditControlsHookForm.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/EditControlsHookForm.tsx @@ -7,7 +7,7 @@ import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; -import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; +import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { HookFormConnectionFormValues } from "./hookFormConfig"; @@ -24,7 +24,7 @@ interface EditControlHookFormProps { * @see EditControls */ export const EditControlsHookForm: React.FC = ({ onCancel }) => { - const { mode, getErrorMessage } = useConnectionHookFormService(); + const { mode, getErrorMessage } = useConnectionFormService(); const { schemaHasBeenRefreshed } = useConnectionEditService(); const { isValid, isDirty, isSubmitting, isSubmitSuccessful, errors } = useFormState(); const { reset, trigger, formState } = useFormContext(); diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionField.tsx deleted file mode 100644 index d87c5e15dee..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionField.tsx +++ /dev/null @@ -1,89 +0,0 @@ -import { Field, FieldProps, useFormikContext } from "formik"; -import { useCallback } from "react"; -import { FormattedMessage } from "react-intl"; - -import { ControlLabels } from "components/LabeledControl"; -import { Button } from "components/ui/Button"; -import { FlexContainer } from "components/ui/Flex"; -import { Text } from "components/ui/Text"; -import { TextInputContainer } from "components/ui/TextInputContainer"; - -import { NamespaceDefinitionType } from "core/request/AirbyteClient"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useModalService } from "hooks/services/Modal"; - -import { FormikConnectionFormValues } from "./formConfig"; -import { FormFieldLayout } from "./FormFieldLayout"; -import { namespaceDefinitionOptions } from "./types"; -import { DestinationNamespaceModal, DestinationNamespaceFormValues } from "../DestinationNamespaceModal"; - -export const NamespaceDefinitionField = () => { - const { mode } = useConnectionFormService(); - const { openModal, closeModal } = useModalService(); - - const formikProps = useFormikContext(); - - const destinationNamespaceHookFormChange = useCallback( - (value: DestinationNamespaceFormValues) => { - formikProps.setFieldValue("namespaceDefinition", value.namespaceDefinition); - - if (value.namespaceDefinition === NamespaceDefinitionType.customformat) { - formikProps.setFieldValue("namespaceFormat", value.namespaceFormat); - } - }, - [formikProps] - ); - - const openDestinationNamespaceModal = useCallback( - () => - openModal({ - size: "lg", - title: , - content: () => ( - - ), - }), - [ - closeModal, - destinationNamespaceHookFormChange, - formikProps.values.namespaceDefinition, - formikProps.values.namespaceFormat, - openModal, - ] - ); - return ( - - {({ field }: FieldProps) => ( - - } - infoTooltipContent={} - /> - - - - - - - - - - )} - - ); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceField.tsx deleted file mode 100644 index cf48c20b7ed..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceField.tsx +++ /dev/null @@ -1,80 +0,0 @@ -import { FieldProps } from "formik"; -import { useMemo } from "react"; -import { FormattedMessage, useIntl } from "react-intl"; - -import { ControlLabels } from "components"; -import { DropDown } from "components/ui/DropDown"; -import { Message } from "components/ui/Message"; - -import { NonBreakingChangesPreference } from "core/request/AirbyteClient"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useExperiment } from "hooks/services/Experiment"; - -import { FormFieldLayout } from "./FormFieldLayout"; - -export const NonBreakingChangesPreferenceField: React.FC> = ({ field, form }) => { - const { connection, mode } = useConnectionFormService(); - const autoPropagationEnabled = useExperiment("autopropagation.enabled", true); - const autoPropagationPrefix = autoPropagationEnabled ? "autopropagation." : ""; - const labelKey = autoPropagationEnabled - ? "connectionForm.nonBreakingChangesPreference.autopropagation.label" - : "connectionForm.nonBreakingChangesPreference.label"; - - const showAutoPropagationMessage = - mode === "edit" && - field.value !== connection.nonBreakingChangesPreference && - (field.value === "propagate_columns" || field.value === "propagate_fully"); - - const supportedPreferences = useMemo(() => { - if (autoPropagationEnabled) { - return [ - NonBreakingChangesPreference.ignore, - NonBreakingChangesPreference.disable, - NonBreakingChangesPreference.propagate_columns, - NonBreakingChangesPreference.propagate_fully, - ]; - } - return [NonBreakingChangesPreference.ignore, NonBreakingChangesPreference.disable]; - }, [autoPropagationEnabled]); - - const { formatMessage } = useIntl(); - - const preferenceOptions = useMemo(() => { - return supportedPreferences.map((value) => ({ - value, - label: formatMessage({ id: `connectionForm.nonBreakingChangesPreference.${autoPropagationPrefix}${value}` }), - testId: `nonBreakingChangesPreference-${value}`, - })); - }, [formatMessage, supportedPreferences, autoPropagationPrefix]); - - return ( - <> - - - form.setFieldValue(field.name, value)} - /> - - {showAutoPropagationMessage && ( - } - /> - )} - - ); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceHookFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceHookFormField.tsx index e481f310dbd..8b311d3b3dc 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceHookFormField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceHookFormField.tsx @@ -6,7 +6,7 @@ import { FormControl } from "components/forms"; import { Message } from "components/ui/Message"; import { NonBreakingChangesPreference } from "core/request/AirbyteClient"; -import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; +import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useExperiment } from "hooks/services/Experiment"; import { HookFormConnectionFormValues } from "./hookFormConfig"; @@ -14,7 +14,7 @@ import { HookFormFieldLayout } from "./HookFormFieldLayout"; export const NonBreakingChangesPreferenceHookFormField = () => { const { formatMessage } = useIntl(); - const { connection, mode } = useConnectionHookFormService(); + const { connection, mode } = useConnectionFormService(); const autoPropagationEnabled = useExperiment("autopropagation.enabled", true); const autoPropagationPrefix = autoPropagationEnabled ? "autopropagation." : ""; const labelKey = autoPropagationEnabled @@ -34,10 +34,10 @@ export const NonBreakingChangesPreferenceHookFormField = () => { const supportedPreferences = useMemo(() => { if (autoPropagationEnabled) { return [ - NonBreakingChangesPreference.ignore, - NonBreakingChangesPreference.disable, NonBreakingChangesPreference.propagate_columns, NonBreakingChangesPreference.propagate_fully, + NonBreakingChangesPreference.ignore, + NonBreakingChangesPreference.disable, ]; } return [NonBreakingChangesPreference.ignore, NonBreakingChangesPreference.disable]; @@ -47,7 +47,7 @@ export const NonBreakingChangesPreferenceHookFormField = () => { return supportedPreferences.map((value) => ({ value, label: formatMessage({ id: `connectionForm.nonBreakingChangesPreference.${autoPropagationPrefix}${value}` }), - testId: `nonBreakingChangesPreference-${value}`, + "data-testid": value, })); }, [formatMessage, supportedPreferences, autoPropagationPrefix]); diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationField.module.scss b/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationField.module.scss deleted file mode 100644 index b5ddcbf3769..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationField.module.scss +++ /dev/null @@ -1,5 +0,0 @@ -@use "scss/variables"; - -.normalizationField { - margin: variables.$spacing-lg 0; -} diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationField.tsx deleted file mode 100644 index 5962b4ff3ad..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationField.tsx +++ /dev/null @@ -1,54 +0,0 @@ -import { FieldProps } from "formik"; -import React from "react"; -import { FormattedMessage } from "react-intl"; - -import { LabeledRadioButton } from "components"; -import { ExternalLink } from "components/ui/Link"; - -import { NormalizationType } from "area/connection/types"; -import { links } from "core/utils/links"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; - -import styles from "./NormalizationField.module.scss"; - -type NormalizationBlockProps = FieldProps; - -/** - * @deprecated will be removed during clean up - https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - * use NormalizationHookForm.tsx instead - * @see NormalizationHookFormField - */ -export const NormalizationField: React.FC = ({ form, field }) => { - const { mode } = useConnectionFormService(); - - return ( -
- } - value={NormalizationType.raw} - checked={field.value === NormalizationType.raw} - disabled={mode === "readonly"} - /> - } - value={NormalizationType.basic} - checked={field.value === NormalizationType.basic} - disabled={mode === "readonly"} - message={ - mode !== "readonly" && ( - {lnk}, - }} - /> - ) - } - /> -
- ); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSection.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSection.tsx deleted file mode 100644 index 57b301be247..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSection.tsx +++ /dev/null @@ -1,70 +0,0 @@ -import { Field, FieldArray } from "formik"; -import React from "react"; -import { useIntl } from "react-intl"; - -import { Card } from "components/ui/Card"; -import { FlexContainer } from "components/ui/Flex"; -import { Heading } from "components/ui/Heading"; - -import { FeatureItem, useFeature } from "core/services/features"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; - -import { NormalizationField } from "./NormalizationField"; -import { TransformationField } from "./TransformationField"; - -interface OperationsSectionProps { - onStartEditTransformation?: () => void; - onEndEditTransformation?: () => void; -} - -/** - * @deprecated will be removed during clean up - https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - * use OperationsSectionHookForm.tsx instead - * @see OperationsSectionHookForm - */ -export const OperationsSection: React.FC = ({ - onStartEditTransformation, - onEndEditTransformation, -}) => { - const { formatMessage } = useIntl(); - - const { - destDefinitionVersion: { normalizationConfig, supportsDbt }, - } = useConnectionFormService(); - - const supportsNormalization = normalizationConfig.supported; - const supportsTransformations = useFeature(FeatureItem.AllowCustomDBT) && supportsDbt; - - if (!supportsNormalization && !supportsTransformations) { - return null; - } - - return ( - - - {supportsNormalization || supportsTransformations ? ( - - {[ - supportsNormalization && formatMessage({ id: "connectionForm.normalization.title" }), - supportsTransformations && formatMessage({ id: "connectionForm.transformation.title" }), - ] - .filter(Boolean) - .join(" & ")} - - ) : null} - {supportsNormalization && } - {supportsTransformations && ( - - {(formProps) => ( - - )} - - )} - - - ); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.module.scss b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.module.scss deleted file mode 100644 index cd6c7eec04d..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.module.scss +++ /dev/null @@ -1,13 +0,0 @@ -@use "scss/colors"; -@use "scss/variables"; - -.connectorLabel { - max-width: 328px; - margin-right: 20px; - vertical-align: top; -} - -.cronZonesDropdown { - margin-left: 10px; - min-width: 150px; -} diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.tsx deleted file mode 100644 index 4cd9dc84228..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.tsx +++ /dev/null @@ -1,235 +0,0 @@ -import { Field, FieldInputProps, FieldProps, FormikProps, useField } from "formik"; -import React, { ChangeEvent, useCallback, useMemo } from "react"; -import { FormattedMessage, useIntl } from "react-intl"; - -import { ControlLabels } from "components"; -import { Box } from "components/ui/Box"; -import { DropDown, DropDownOptionDataItem } from "components/ui/DropDown"; -import { FlexContainer } from "components/ui/Flex"; -import { Input } from "components/ui/Input"; -import { ExternalLink } from "components/ui/Link"; -import { Text } from "components/ui/Text"; - -import { ConnectionScheduleData, ConnectionScheduleType } from "core/request/AirbyteClient"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; -import { isCloudApp } from "core/utils/app"; -import { links } from "core/utils/links"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; - -import availableCronTimeZones from "./availableCronTimeZones.json"; -import { FormikConnectionFormValues, useFrequencyDropdownData } from "./formConfig"; -import { FormFieldLayout } from "./FormFieldLayout"; -import styles from "./ScheduleField.module.scss"; - -const CRON_DEFAULT_VALUE = { - cronTimeZone: "UTC", - // Fire at 12:00 PM (noon) every day - cronExpression: "0 0 12 * * ?", -}; - -const CronErrorChatWithUsButton: React.FC> = ({ children }) => { - return {children}; -}; - -export const ScheduleField: React.FC = () => { - const { formatMessage } = useIntl(); - const { connection, mode } = useConnectionFormService(); - const frequencies = useFrequencyDropdownData(connection.scheduleData); - const analyticsService = useAnalyticsService(); - - const onDropDownSelect = useCallback( - (item: DropDownOptionDataItem | null) => { - const enabledStreams = connection.syncCatalog.streams.filter((stream) => stream.config?.selected).length; - - if (item) { - analyticsService.track(Namespace.CONNECTION, Action.FREQUENCY, { - actionDescription: "Frequency selected", - frequency: item.label, - connector_source_definition: connection.source.sourceName, - connector_source_definition_id: connection.source.sourceDefinitionId, - connector_destination_definition: connection.destination.destinationName, - connector_destination_definition_id: connection.destination.destinationDefinitionId, - available_streams: connection.syncCatalog.streams.length, - enabled_streams: enabledStreams, - type: mode, - }); - } - }, - [ - analyticsService, - connection.destination.destinationDefinitionId, - connection.destination.destinationName, - connection.source.sourceDefinitionId, - connection.source.sourceName, - connection.syncCatalog.streams, - mode, - ] - ); - - const onScheduleChange = (item: DropDownOptionDataItem, form: FormikProps) => { - onDropDownSelect?.(item); - - let scheduleData: ConnectionScheduleData; - const isManual = item.value === ConnectionScheduleType.manual; - const isCron = item.value === ConnectionScheduleType.cron; - - // Set scheduleType for yup validation - const scheduleType = isManual || isCron ? (item.value as ConnectionScheduleType) : ConnectionScheduleType.basic; - - // Set scheduleData.basicSchedule - if (isManual || isCron) { - scheduleData = { - basicSchedule: undefined, - cron: isCron ? CRON_DEFAULT_VALUE : undefined, - }; - } else { - scheduleData = { - basicSchedule: item.value, - }; - } - - form.setValues( - { - ...form.values, - scheduleType, - scheduleData, - }, - true - ); - }; - - const getBasicScheduleValue = (value: ConnectionScheduleData, form: FormikProps) => { - const { scheduleType } = form.values; - - if (scheduleType === ConnectionScheduleType.basic) { - return value.basicSchedule; - } - - if (!scheduleType) { - return null; - } - - return formatMessage({ - id: `frequency.${scheduleType}`, - }).toLowerCase(); - }; - - const getZoneValue = (currentSelectedZone = "UTC") => currentSelectedZone; - - const onCronChange = ( - event: DropDownOptionDataItem | ChangeEvent, - field: FieldInputProps, - form: FormikProps, - key: string - ) => { - form.setFieldValue(field.name, { - cron: { - ...field.value?.cron, - [key]: (event as DropDownOptionDataItem).value - ? (event as DropDownOptionDataItem).value - : (event as ChangeEvent).currentTarget.value, - }, - }); - }; - - const cronTimeZones = useMemo(() => { - return availableCronTimeZones.map((zone: string) => ({ label: zone, value: zone })); - }, []); - - const isCron = (form: FormikProps): boolean => { - return form.values.scheduleType === ConnectionScheduleType.cron; - }; - - const [, { error: cronValidationError }] = useField("scheduleData.cron.cronExpression"); - - return ( - - {({ field, meta, form }: FieldProps) => ( - <> - - - { - onScheduleChange(item, form); - }} - value={getBasicScheduleValue(field.value, form)} - isDisabled={form.isSubmitting || mode === "readonly"} - /> - - {isCron(form) && ( - - {lnk}, - } - )} - /> - - ) => - onCronChange(event, field, form, "cronExpression") - } - /> - onCronChange(item, field, form, "cronTimeZone")} - /> - - {cronValidationError && ( - - - ( - {btnText} - ), - }, - } - : {})} - /> - - - )} - - )} - - )} - - ); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/BasicScheduleFormControl.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/BasicScheduleFormControl.tsx index c85cf3a9e32..006aff4a8e3 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/BasicScheduleFormControl.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/BasicScheduleFormControl.tsx @@ -6,7 +6,7 @@ import { FormLabel } from "components/forms/FormControl"; import { ListBox, Option } from "components/ui/ListBox"; import { ConnectionScheduleDataBasicSchedule } from "core/request/AirbyteClient"; -import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; +import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useBasicFrequencyDropdownDataHookForm } from "./useBasicFrequencyDropdownDataHookForm"; import { useTrackConnectionFrequency } from "./useTrackConnectionFrequency"; @@ -15,7 +15,7 @@ import { HookFormConnectionFormValues } from "../hookFormConfig"; export const BasicScheduleFormControl: React.FC = () => { const { formatMessage } = useIntl(); - const { connection } = useConnectionHookFormService(); + const { connection } = useConnectionFormService(); const { setValue, control } = useFormContext(); const { trackDropdownSelect } = useTrackConnectionFrequency(connection); const frequencies: Array> = useBasicFrequencyDropdownDataHookForm( diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/ScheduleTypeFormControl.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/ScheduleTypeFormControl.tsx index 06d51fa15f7..f47a5b86085 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/ScheduleTypeFormControl.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/ScheduleTypeFormControl.tsx @@ -22,18 +22,21 @@ export const ScheduleTypeFormControl: React.FC = () => { id: "frequency.scheduled", }), value: ConnectionScheduleType.basic, + "data-testid": "scheduled", }, { label: formatMessage({ id: "frequency.manual", }), value: ConnectionScheduleType.manual, + "data-testid": "manual", }, { label: formatMessage({ id: "frequency.cron", }), value: ConnectionScheduleType.cron, + "data-testid": "cron", }, ]; @@ -74,6 +77,7 @@ export const ScheduleTypeFormControl: React.FC = () => { onScheduleTypeSelect(value); }} selectedValue={field.value} + data-testid="schedule-type" /> )} diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/useBasicFrequencyDropdownDataHookForm.test.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/useBasicFrequencyDropdownDataHookForm.test.tsx index c73e666609e..118376567ca 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/useBasicFrequencyDropdownDataHookForm.test.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/useBasicFrequencyDropdownDataHookForm.test.tsx @@ -4,8 +4,7 @@ import { TestWrapper as wrapper } from "test-utils/testutils"; import { ConnectionScheduleTimeUnit } from "core/request/AirbyteClient"; -import { useBasicFrequencyDropdownDataHookForm } from "./useBasicFrequencyDropdownDataHookForm"; -import { frequencyConfig } from "./useBasicFrequencyDropdownDataHookForm"; +import { useBasicFrequencyDropdownDataHookForm, frequencyConfig } from "./useBasicFrequencyDropdownDataHookForm"; describe("#useBasicFrequencyDropdownDataHookForm", () => { it("should return only default frequencies when no additional frequency is provided", () => { diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/useTrackConnectionFrequency.ts b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/useTrackConnectionFrequency.ts index df6a19eb698..09ec376cef8 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/useTrackConnectionFrequency.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleHookFormField/useTrackConnectionFrequency.ts @@ -3,13 +3,13 @@ import { useCallback } from "react"; import { ConnectionScheduleDataBasicSchedule } from "core/request/AirbyteClient"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { + useConnectionFormService, ConnectionOrPartialConnection, - useConnectionHookFormService, -} from "hooks/services/ConnectionForm/ConnectionHookFormService"; +} from "hooks/services/ConnectionForm/ConnectionFormService"; export const useTrackConnectionFrequency = (connection: ConnectionOrPartialConnection) => { const analyticsService = useAnalyticsService(); - const { mode } = useConnectionHookFormService(); + const { mode } = useConnectionFormService(); const trackDropdownSelect = useCallback( (value: ConnectionScheduleDataBasicSchedule) => { diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.module.scss b/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.module.scss deleted file mode 100644 index 289e33688b7..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.module.scss +++ /dev/null @@ -1,13 +0,0 @@ -@use "scss/variables"; - -.header { - margin: variables.$spacing-md 0 0; - display: flex; - justify-content: space-between; - align-items: center; - flex-direction: row; - font-weight: 500; - font-size: variables.$font-size-lg; - line-height: 1.2; - padding: variables.$spacing-md variables.$spacing-xl; -} diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.tsx deleted file mode 100644 index 015c9e5bd86..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogField.tsx +++ /dev/null @@ -1,57 +0,0 @@ -import { FieldProps } from "formik"; -import React, { useCallback } from "react"; -import { FormattedMessage } from "react-intl"; - -import { SyncCatalog } from "components/connection/syncCatalog/SyncCatalog"; -import { Heading } from "components/ui/Heading"; - -import { SyncSchemaStream } from "core/domain/catalog"; -import { DestinationSyncMode } from "core/request/AirbyteClient"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; - -import styles from "./SyncCatalogField.module.scss"; - -interface SchemaViewProps extends FieldProps { - additionalControl?: React.ReactNode; - destinationSupportedSyncModes: DestinationSyncMode[]; - isSubmitting: boolean; -} - -/** - * @deprecated will be removed during clean up - https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - * use SyncCatalogHookFormField.tsx instead - * @see SyncCatalogHookFormField - */ -const SyncCatalogFieldComponent: React.FC> = ({ - additionalControl, - field, - form, - isSubmitting, -}) => { - const { mode } = useConnectionFormService(); - - const { value: streams, name: fieldName } = field; - - const setField = form.setFieldValue; - - const onStreamsUpdated = useCallback( - (newValue: SyncSchemaStream[]) => { - setField(fieldName, newValue); - }, - [fieldName, setField] - ); - - return ( - <> -
- - - - {mode !== "readonly" && additionalControl} -
- - - ); -}; - -export const SyncCatalogField = React.memo(SyncCatalogFieldComponent); diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogHookFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogHookFormField.tsx index 395a37beb92..9e79063c77e 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogHookFormField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogHookFormField.tsx @@ -22,7 +22,6 @@ import { useRefreshSourceSchemaWithConfirmationOnDirty } from "./refreshSourceSc import styles from "./SyncCatalogHookFormField.module.scss"; import { StreamsConfigTableHeaderHookForm } from "../syncCatalog/StreamsConfigTable/StreamsConfigTableHeaderHookForm"; import { DisabledStreamsSwitch } from "../syncCatalog/SyncCatalog/DisabledStreamsSwitch"; -import { LocationWithState } from "../syncCatalog/SyncCatalog/SyncCatalogBody"; import { SyncCatalogEmpty } from "../syncCatalog/SyncCatalog/SyncCatalogEmpty"; import { SyncCatalogRowHookForm } from "../syncCatalog/SyncCatalog/SyncCatalogRowHookForm"; import { SyncCatalogStreamSearch } from "../syncCatalog/SyncCatalog/SyncCatalogStreamSearch"; @@ -79,7 +78,7 @@ export const SyncCatalogHookFormField: React.FC = () => { ); // Scroll to the stream that was redirected from the Status tab - const { state: locationState } = useLocation() as LocationWithState; + const { state: locationState } = useLocation() as LocationWithStateHookForm; const initialTopMostItemIndex: IndexLocationWithAlign | undefined = useMemo(() => { if (locationState?.action !== "showInReplicationTable" && locationState?.action !== "openDetails") { return; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/TransformationField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/TransformationField.tsx deleted file mode 100644 index d346df8a573..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/TransformationField.tsx +++ /dev/null @@ -1,81 +0,0 @@ -import { ArrayHelpers, FormikProps } from "formik"; -import React, { useState } from "react"; -import { FormattedMessage } from "react-intl"; - -import { ArrayOfObjectsEditor } from "components/ArrayOfObjectsEditor"; -import TransformationForm from "components/connection/TransformationForm"; - -import { OperationRead } from "core/request/AirbyteClient"; -import { isDefined } from "core/utils/common"; -import { ConnectionFormMode } from "hooks/services/ConnectionForm/ConnectionFormService"; - -import { useDefaultTransformation } from "./formConfig"; - -interface TransformationFieldProps extends ArrayHelpers { - form: FormikProps<{ transformations: OperationRead[] }>; - mode?: ConnectionFormMode; - onStartEdit?: () => void; - onEndEdit?: () => void; -} - -/** - * @deprecated will be removed during clean up - https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - * use TransformationHookForm.tsx instead - * @see TransformationFieldHookForm - */ -const TransformationField: React.FC = ({ - remove, - push, - replace, - form, - mode, - onStartEdit, - onEndEdit, -}) => { - const [editableItemIdx, setEditableItem] = useState(null); - const defaultTransformation = useDefaultTransformation(); - const clearEditableItem = () => setEditableItem(null); - - return ( - - } - addButtonText={} - onRemove={remove} - onStartEdit={(idx) => { - setEditableItem(idx); - onStartEdit?.(); - }} - onCancel={() => { - clearEditableItem(); - onEndEdit?.(); - }} - mode={mode} - editModalSize="xl" - renderItemEditorForm={(editableItem) => ( - { - clearEditableItem(); - onEndEdit?.(); - }} - onDone={(transformation) => { - if (isDefined(editableItemIdx)) { - editableItemIdx >= form.values.transformations.length - ? push(transformation) - : replace(editableItemIdx, transformation); - clearEditableItem(); - onEndEdit?.(); - } - }} - /> - )} - /> - ); -}; - -export { TransformationField }; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/TransformationFieldHookForm.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/TransformationFieldHookForm.tsx index dbe4ee81e93..c32b1349016 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/TransformationFieldHookForm.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/TransformationFieldHookForm.tsx @@ -1,28 +1,42 @@ -import React from "react"; +import React, { useMemo } from "react"; import { useFieldArray } from "react-hook-form"; import { FormattedMessage } from "react-intl"; import { ArrayOfObjectsHookFormEditor } from "components/ArrayOfObjectsEditor"; +import { OperationCreate, OperatorType } from "core/api/types/AirbyteClient"; import { isDefined } from "core/utils/common"; import { useModalService } from "hooks/services/Modal"; +import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import { CustomTransformationsFormValues } from "pages/connections/ConnectionTransformationPage/CustomTransformationsForm"; -import { useDefaultTransformation } from "./formConfig"; import { DbtOperationReadOrCreate, TransformationHookForm } from "../TransformationHookForm"; /** - * Custom transformations field for react-hook-form - * will replace TransformationField in the future - * @see TransformationField - * @constructor + * react-hook-form custom transformations form */ export const TransformationFieldHookForm: React.FC = () => { + const { workspaceId } = useCurrentWorkspace(); const { fields, append, remove, update } = useFieldArray({ name: "transformations", }); const { openModal, closeModal } = useModalService(); - const defaultTransformation = useDefaultTransformation(); + + const defaultTransformation: OperationCreate = useMemo( + () => ({ + name: "My dbt transformations", + workspaceId, + operatorConfiguration: { + operatorType: OperatorType.dbt, + dbt: { + gitRepoUrl: "", + dockerImage: "fishtownanalytics/dbt:1.0.0", + dbtArguments: "run", + }, + }, + }), + [workspaceId] + ); const openEditModal = (transformationItemIndex?: number) => openModal({ diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/__snapshots__/formConfig.test.ts.snap b/airbyte-webapp/src/components/connection/ConnectionForm/__snapshots__/formConfig.test.ts.snap deleted file mode 100644 index 249a6f9767d..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/__snapshots__/formConfig.test.ts.snap +++ /dev/null @@ -1,4692 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`#useInitialValues should generate initial values w/ 'not create' mode: false 1`] = ` -{ - "geography": "auto", - "name": "Scrafty <> Heroku Postgres", - "namespaceDefinition": "source", - "namespaceFormat": "\${SOURCE_NAMESPACE}", - "nonBreakingChangesPreference": "ignore", - "normalization": "basic", - "prefix": "", - "scheduleData": null, - "scheduleType": "manual", - "syncCatalog": { - "streams": [ - { - "config": { - "aliasName": "pokemon", - "cursorField": [], - "destinationSyncMode": "overwrite", - "primaryKey": [], - "selected": true, - "syncMode": "full_refresh", - }, - "id": "0", - "stream": { - "defaultCursorField": [], - "jsonSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "abilities": { - "items": { - "properties": { - "ability": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "is_hidden": { - "type": [ - "null", - "boolean", - ], - }, - "slot": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "base_experience": { - "type": [ - "null", - "integer", - ], - }, - "forms": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "game_indices": { - "items": { - "properties": { - "game_index": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "height": { - "type": [ - "null", - "integer", - ], - }, - "held_items": { - "items": { - "properties": { - "item": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_details": { - "items": { - "properties": { - "rarity": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "id": { - "type": [ - "null", - "integer", - ], - }, - "is_default ": { - "type": [ - "null", - "boolean", - ], - }, - "location_area_encounters": { - "type": [ - "null", - "string", - ], - }, - "moves": { - "items": { - "properties": { - "move": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group_details": { - "items": { - "properties": { - "level_learned_at": { - "type": [ - "null", - "integer", - ], - }, - "move_learn_method": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "name": { - "type": [ - "null", - "string", - ], - }, - "order": { - "type": [ - "null", - "integer", - ], - }, - "species": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "sprites": { - "properties": { - "back_default": { - "type": [ - "null", - "string", - ], - }, - "back_female": { - "type": [ - "null", - "string", - ], - }, - "back_shiny": { - "type": [ - "null", - "string", - ], - }, - "back_shiny_female": { - "type": [ - "null", - "string", - ], - }, - "front_default": { - "type": [ - "null", - "string", - ], - }, - "front_female": { - "type": [ - "null", - "string", - ], - }, - "front_shiny": { - "type": [ - "null", - "string", - ], - }, - "front_shiny_female": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "stats": { - "items": { - "properties": { - "base_stat": { - "type": [ - "null", - "integer", - ], - }, - "effort": { - "type": [ - "null", - "integer", - ], - }, - "stat": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "types": { - "items": { - "properties": { - "slot": { - "type": [ - "null", - "integer", - ], - }, - "type": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "weight": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": "object", - }, - "name": "pokemon", - "sourceDefinedPrimaryKey": [], - "supportedSyncModes": [ - "full_refresh", - ], - }, - }, - { - "config": { - "aliasName": "pokemon", - "cursorField": [], - "destinationSyncMode": "overwrite", - "primaryKey": [], - "selected": false, - "syncMode": "full_refresh", - }, - "id": "1", - "stream": { - "defaultCursorField": [], - "jsonSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "abilities": { - "items": { - "properties": { - "ability": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "is_hidden": { - "type": [ - "null", - "boolean", - ], - }, - "slot": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "base_experience": { - "type": [ - "null", - "integer", - ], - }, - "forms": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "game_indices": { - "items": { - "properties": { - "game_index": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "height": { - "type": [ - "null", - "integer", - ], - }, - "held_items": { - "items": { - "properties": { - "item": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_details": { - "items": { - "properties": { - "rarity": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "id": { - "type": [ - "null", - "integer", - ], - }, - "is_default ": { - "type": [ - "null", - "boolean", - ], - }, - "location_area_encounters": { - "type": [ - "null", - "string", - ], - }, - "moves": { - "items": { - "properties": { - "move": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group_details": { - "items": { - "properties": { - "level_learned_at": { - "type": [ - "null", - "integer", - ], - }, - "move_learn_method": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "name": { - "type": [ - "null", - "string", - ], - }, - "order": { - "type": [ - "null", - "integer", - ], - }, - "species": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "sprites": { - "properties": { - "back_default": { - "type": [ - "null", - "string", - ], - }, - "back_female": { - "type": [ - "null", - "string", - ], - }, - "back_shiny": { - "type": [ - "null", - "string", - ], - }, - "back_shiny_female": { - "type": [ - "null", - "string", - ], - }, - "front_default": { - "type": [ - "null", - "string", - ], - }, - "front_female": { - "type": [ - "null", - "string", - ], - }, - "front_shiny": { - "type": [ - "null", - "string", - ], - }, - "front_shiny_female": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "stats": { - "items": { - "properties": { - "base_stat": { - "type": [ - "null", - "integer", - ], - }, - "effort": { - "type": [ - "null", - "integer", - ], - }, - "stat": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "types": { - "items": { - "properties": { - "slot": { - "type": [ - "null", - "integer", - ], - }, - "type": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "weight": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": "object", - }, - "name": "pokemon2", - "sourceDefinedPrimaryKey": [], - "supportedSyncModes": [ - "full_refresh", - ], - }, - }, - { - "config": { - "aliasName": "pokemon", - "cursorField": [], - "destinationSyncMode": "overwrite", - "primaryKey": [], - "selected": true, - "syncMode": "full_refresh", - }, - "id": "2", - "stream": { - "defaultCursorField": [], - "jsonSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "abilities": { - "items": { - "properties": { - "ability": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "is_hidden": { - "type": [ - "null", - "boolean", - ], - }, - "slot": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "base_experience": { - "type": [ - "null", - "integer", - ], - }, - "forms": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "game_indices": { - "items": { - "properties": { - "game_index": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "height": { - "type": [ - "null", - "integer", - ], - }, - "held_items": { - "items": { - "properties": { - "item": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_details": { - "items": { - "properties": { - "rarity": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "id": { - "type": [ - "null", - "integer", - ], - }, - "is_default ": { - "type": [ - "null", - "boolean", - ], - }, - "location_area_encounters": { - "type": [ - "null", - "string", - ], - }, - "moves": { - "items": { - "properties": { - "move": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group_details": { - "items": { - "properties": { - "level_learned_at": { - "type": [ - "null", - "integer", - ], - }, - "move_learn_method": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "name": { - "type": [ - "null", - "string", - ], - }, - "order": { - "type": [ - "null", - "integer", - ], - }, - "species": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "sprites": { - "properties": { - "back_default": { - "type": [ - "null", - "string", - ], - }, - "back_female": { - "type": [ - "null", - "string", - ], - }, - "back_shiny": { - "type": [ - "null", - "string", - ], - }, - "back_shiny_female": { - "type": [ - "null", - "string", - ], - }, - "front_default": { - "type": [ - "null", - "string", - ], - }, - "front_female": { - "type": [ - "null", - "string", - ], - }, - "front_shiny": { - "type": [ - "null", - "string", - ], - }, - "front_shiny_female": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "stats": { - "items": { - "properties": { - "base_stat": { - "type": [ - "null", - "integer", - ], - }, - "effort": { - "type": [ - "null", - "integer", - ], - }, - "stat": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "types": { - "items": { - "properties": { - "slot": { - "type": [ - "null", - "integer", - ], - }, - "type": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "weight": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": "object", - }, - "name": "another_stream", - "sourceDefinedPrimaryKey": [], - "supportedSyncModes": [ - "full_refresh", - ], - }, - }, - ], - }, - "transformations": [], -} -`; - -exports[`#useInitialValues should generate initial values w/ 'not create' mode: true 1`] = ` -{ - "geography": "auto", - "namespaceDefinition": "source", - "namespaceFormat": "\${SOURCE_NAMESPACE}", - "nonBreakingChangesPreference": "ignore", - "normalization": "basic", - "prefix": "", - "scheduleData": null, - "scheduleType": "manual", - "syncCatalog": { - "streams": [ - { - "config": { - "aliasName": "pokemon", - "cursorField": [], - "destinationSyncMode": "append", - "primaryKey": [], - "selected": true, - "syncMode": "full_refresh", - }, - "id": "0", - "stream": { - "defaultCursorField": [], - "jsonSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "abilities": { - "items": { - "properties": { - "ability": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "is_hidden": { - "type": [ - "null", - "boolean", - ], - }, - "slot": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "base_experience": { - "type": [ - "null", - "integer", - ], - }, - "forms": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "game_indices": { - "items": { - "properties": { - "game_index": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "height": { - "type": [ - "null", - "integer", - ], - }, - "held_items": { - "items": { - "properties": { - "item": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_details": { - "items": { - "properties": { - "rarity": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "id": { - "type": [ - "null", - "integer", - ], - }, - "is_default ": { - "type": [ - "null", - "boolean", - ], - }, - "location_area_encounters": { - "type": [ - "null", - "string", - ], - }, - "moves": { - "items": { - "properties": { - "move": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group_details": { - "items": { - "properties": { - "level_learned_at": { - "type": [ - "null", - "integer", - ], - }, - "move_learn_method": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "name": { - "type": [ - "null", - "string", - ], - }, - "order": { - "type": [ - "null", - "integer", - ], - }, - "species": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "sprites": { - "properties": { - "back_default": { - "type": [ - "null", - "string", - ], - }, - "back_female": { - "type": [ - "null", - "string", - ], - }, - "back_shiny": { - "type": [ - "null", - "string", - ], - }, - "back_shiny_female": { - "type": [ - "null", - "string", - ], - }, - "front_default": { - "type": [ - "null", - "string", - ], - }, - "front_female": { - "type": [ - "null", - "string", - ], - }, - "front_shiny": { - "type": [ - "null", - "string", - ], - }, - "front_shiny_female": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "stats": { - "items": { - "properties": { - "base_stat": { - "type": [ - "null", - "integer", - ], - }, - "effort": { - "type": [ - "null", - "integer", - ], - }, - "stat": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "types": { - "items": { - "properties": { - "slot": { - "type": [ - "null", - "integer", - ], - }, - "type": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "weight": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": "object", - }, - "name": "pokemon", - "sourceDefinedPrimaryKey": [], - "supportedSyncModes": [ - "full_refresh", - ], - }, - }, - { - "config": { - "aliasName": "pokemon", - "cursorField": [], - "destinationSyncMode": "append", - "primaryKey": [], - "selected": false, - "syncMode": "full_refresh", - }, - "id": "1", - "stream": { - "defaultCursorField": [], - "jsonSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "abilities": { - "items": { - "properties": { - "ability": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "is_hidden": { - "type": [ - "null", - "boolean", - ], - }, - "slot": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "base_experience": { - "type": [ - "null", - "integer", - ], - }, - "forms": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "game_indices": { - "items": { - "properties": { - "game_index": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "height": { - "type": [ - "null", - "integer", - ], - }, - "held_items": { - "items": { - "properties": { - "item": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_details": { - "items": { - "properties": { - "rarity": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "id": { - "type": [ - "null", - "integer", - ], - }, - "is_default ": { - "type": [ - "null", - "boolean", - ], - }, - "location_area_encounters": { - "type": [ - "null", - "string", - ], - }, - "moves": { - "items": { - "properties": { - "move": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group_details": { - "items": { - "properties": { - "level_learned_at": { - "type": [ - "null", - "integer", - ], - }, - "move_learn_method": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "name": { - "type": [ - "null", - "string", - ], - }, - "order": { - "type": [ - "null", - "integer", - ], - }, - "species": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "sprites": { - "properties": { - "back_default": { - "type": [ - "null", - "string", - ], - }, - "back_female": { - "type": [ - "null", - "string", - ], - }, - "back_shiny": { - "type": [ - "null", - "string", - ], - }, - "back_shiny_female": { - "type": [ - "null", - "string", - ], - }, - "front_default": { - "type": [ - "null", - "string", - ], - }, - "front_female": { - "type": [ - "null", - "string", - ], - }, - "front_shiny": { - "type": [ - "null", - "string", - ], - }, - "front_shiny_female": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "stats": { - "items": { - "properties": { - "base_stat": { - "type": [ - "null", - "integer", - ], - }, - "effort": { - "type": [ - "null", - "integer", - ], - }, - "stat": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "types": { - "items": { - "properties": { - "slot": { - "type": [ - "null", - "integer", - ], - }, - "type": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "weight": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": "object", - }, - "name": "pokemon2", - "sourceDefinedPrimaryKey": [], - "supportedSyncModes": [ - "full_refresh", - ], - }, - }, - { - "config": { - "aliasName": "pokemon", - "cursorField": [], - "destinationSyncMode": "append", - "primaryKey": [], - "selected": true, - "syncMode": "full_refresh", - }, - "id": "2", - "stream": { - "defaultCursorField": [], - "jsonSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "abilities": { - "items": { - "properties": { - "ability": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "is_hidden": { - "type": [ - "null", - "boolean", - ], - }, - "slot": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "base_experience": { - "type": [ - "null", - "integer", - ], - }, - "forms": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "game_indices": { - "items": { - "properties": { - "game_index": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "height": { - "type": [ - "null", - "integer", - ], - }, - "held_items": { - "items": { - "properties": { - "item": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_details": { - "items": { - "properties": { - "rarity": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "id": { - "type": [ - "null", - "integer", - ], - }, - "is_default ": { - "type": [ - "null", - "boolean", - ], - }, - "location_area_encounters": { - "type": [ - "null", - "string", - ], - }, - "moves": { - "items": { - "properties": { - "move": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group_details": { - "items": { - "properties": { - "level_learned_at": { - "type": [ - "null", - "integer", - ], - }, - "move_learn_method": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "name": { - "type": [ - "null", - "string", - ], - }, - "order": { - "type": [ - "null", - "integer", - ], - }, - "species": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "sprites": { - "properties": { - "back_default": { - "type": [ - "null", - "string", - ], - }, - "back_female": { - "type": [ - "null", - "string", - ], - }, - "back_shiny": { - "type": [ - "null", - "string", - ], - }, - "back_shiny_female": { - "type": [ - "null", - "string", - ], - }, - "front_default": { - "type": [ - "null", - "string", - ], - }, - "front_female": { - "type": [ - "null", - "string", - ], - }, - "front_shiny": { - "type": [ - "null", - "string", - ], - }, - "front_shiny_female": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "stats": { - "items": { - "properties": { - "base_stat": { - "type": [ - "null", - "integer", - ], - }, - "effort": { - "type": [ - "null", - "integer", - ], - }, - "stat": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "types": { - "items": { - "properties": { - "slot": { - "type": [ - "null", - "integer", - ], - }, - "type": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "weight": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": "object", - }, - "name": "another_stream", - "sourceDefinedPrimaryKey": [], - "supportedSyncModes": [ - "full_refresh", - ], - }, - }, - ], - }, - "transformations": [], -} -`; - -exports[`#useInitialValues should generate initial values w/ no 'not create' mode 1`] = ` -{ - "geography": "auto", - "name": "Scrafty <> Heroku Postgres", - "namespaceDefinition": "source", - "namespaceFormat": "\${SOURCE_NAMESPACE}", - "nonBreakingChangesPreference": "ignore", - "normalization": "basic", - "prefix": "", - "scheduleData": null, - "scheduleType": "manual", - "syncCatalog": { - "streams": [ - { - "config": { - "aliasName": "pokemon", - "cursorField": [], - "destinationSyncMode": "overwrite", - "primaryKey": [], - "selected": true, - "syncMode": "full_refresh", - }, - "id": "0", - "stream": { - "defaultCursorField": [], - "jsonSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "abilities": { - "items": { - "properties": { - "ability": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "is_hidden": { - "type": [ - "null", - "boolean", - ], - }, - "slot": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "base_experience": { - "type": [ - "null", - "integer", - ], - }, - "forms": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "game_indices": { - "items": { - "properties": { - "game_index": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "height": { - "type": [ - "null", - "integer", - ], - }, - "held_items": { - "items": { - "properties": { - "item": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_details": { - "items": { - "properties": { - "rarity": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "id": { - "type": [ - "null", - "integer", - ], - }, - "is_default ": { - "type": [ - "null", - "boolean", - ], - }, - "location_area_encounters": { - "type": [ - "null", - "string", - ], - }, - "moves": { - "items": { - "properties": { - "move": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group_details": { - "items": { - "properties": { - "level_learned_at": { - "type": [ - "null", - "integer", - ], - }, - "move_learn_method": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "name": { - "type": [ - "null", - "string", - ], - }, - "order": { - "type": [ - "null", - "integer", - ], - }, - "species": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "sprites": { - "properties": { - "back_default": { - "type": [ - "null", - "string", - ], - }, - "back_female": { - "type": [ - "null", - "string", - ], - }, - "back_shiny": { - "type": [ - "null", - "string", - ], - }, - "back_shiny_female": { - "type": [ - "null", - "string", - ], - }, - "front_default": { - "type": [ - "null", - "string", - ], - }, - "front_female": { - "type": [ - "null", - "string", - ], - }, - "front_shiny": { - "type": [ - "null", - "string", - ], - }, - "front_shiny_female": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "stats": { - "items": { - "properties": { - "base_stat": { - "type": [ - "null", - "integer", - ], - }, - "effort": { - "type": [ - "null", - "integer", - ], - }, - "stat": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "types": { - "items": { - "properties": { - "slot": { - "type": [ - "null", - "integer", - ], - }, - "type": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "weight": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": "object", - }, - "name": "pokemon", - "sourceDefinedPrimaryKey": [], - "supportedSyncModes": [ - "full_refresh", - ], - }, - }, - { - "config": { - "aliasName": "pokemon", - "cursorField": [], - "destinationSyncMode": "overwrite", - "primaryKey": [], - "selected": false, - "syncMode": "full_refresh", - }, - "id": "1", - "stream": { - "defaultCursorField": [], - "jsonSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "abilities": { - "items": { - "properties": { - "ability": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "is_hidden": { - "type": [ - "null", - "boolean", - ], - }, - "slot": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "base_experience": { - "type": [ - "null", - "integer", - ], - }, - "forms": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "game_indices": { - "items": { - "properties": { - "game_index": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "height": { - "type": [ - "null", - "integer", - ], - }, - "held_items": { - "items": { - "properties": { - "item": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_details": { - "items": { - "properties": { - "rarity": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "id": { - "type": [ - "null", - "integer", - ], - }, - "is_default ": { - "type": [ - "null", - "boolean", - ], - }, - "location_area_encounters": { - "type": [ - "null", - "string", - ], - }, - "moves": { - "items": { - "properties": { - "move": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group_details": { - "items": { - "properties": { - "level_learned_at": { - "type": [ - "null", - "integer", - ], - }, - "move_learn_method": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "name": { - "type": [ - "null", - "string", - ], - }, - "order": { - "type": [ - "null", - "integer", - ], - }, - "species": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "sprites": { - "properties": { - "back_default": { - "type": [ - "null", - "string", - ], - }, - "back_female": { - "type": [ - "null", - "string", - ], - }, - "back_shiny": { - "type": [ - "null", - "string", - ], - }, - "back_shiny_female": { - "type": [ - "null", - "string", - ], - }, - "front_default": { - "type": [ - "null", - "string", - ], - }, - "front_female": { - "type": [ - "null", - "string", - ], - }, - "front_shiny": { - "type": [ - "null", - "string", - ], - }, - "front_shiny_female": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "stats": { - "items": { - "properties": { - "base_stat": { - "type": [ - "null", - "integer", - ], - }, - "effort": { - "type": [ - "null", - "integer", - ], - }, - "stat": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "types": { - "items": { - "properties": { - "slot": { - "type": [ - "null", - "integer", - ], - }, - "type": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "weight": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": "object", - }, - "name": "pokemon2", - "sourceDefinedPrimaryKey": [], - "supportedSyncModes": [ - "full_refresh", - ], - }, - }, - { - "config": { - "aliasName": "pokemon", - "cursorField": [], - "destinationSyncMode": "overwrite", - "primaryKey": [], - "selected": true, - "syncMode": "full_refresh", - }, - "id": "2", - "stream": { - "defaultCursorField": [], - "jsonSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "abilities": { - "items": { - "properties": { - "ability": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "is_hidden": { - "type": [ - "null", - "boolean", - ], - }, - "slot": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "base_experience": { - "type": [ - "null", - "integer", - ], - }, - "forms": { - "items": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "game_indices": { - "items": { - "properties": { - "game_index": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "height": { - "type": [ - "null", - "integer", - ], - }, - "held_items": { - "items": { - "properties": { - "item": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_details": { - "items": { - "properties": { - "rarity": { - "type": [ - "null", - "integer", - ], - }, - "version": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "id": { - "type": [ - "null", - "integer", - ], - }, - "is_default ": { - "type": [ - "null", - "boolean", - ], - }, - "location_area_encounters": { - "type": [ - "null", - "string", - ], - }, - "moves": { - "items": { - "properties": { - "move": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group_details": { - "items": { - "properties": { - "level_learned_at": { - "type": [ - "null", - "integer", - ], - }, - "move_learn_method": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "version_group": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "name": { - "type": [ - "null", - "string", - ], - }, - "order": { - "type": [ - "null", - "integer", - ], - }, - "species": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "sprites": { - "properties": { - "back_default": { - "type": [ - "null", - "string", - ], - }, - "back_female": { - "type": [ - "null", - "string", - ], - }, - "back_shiny": { - "type": [ - "null", - "string", - ], - }, - "back_shiny_female": { - "type": [ - "null", - "string", - ], - }, - "front_default": { - "type": [ - "null", - "string", - ], - }, - "front_female": { - "type": [ - "null", - "string", - ], - }, - "front_shiny": { - "type": [ - "null", - "string", - ], - }, - "front_shiny_female": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "stats": { - "items": { - "properties": { - "base_stat": { - "type": [ - "null", - "integer", - ], - }, - "effort": { - "type": [ - "null", - "integer", - ], - }, - "stat": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "types": { - "items": { - "properties": { - "slot": { - "type": [ - "null", - "integer", - ], - }, - "type": { - "properties": { - "name": { - "type": [ - "null", - "string", - ], - }, - "url": { - "type": [ - "null", - "string", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - }, - "type": [ - "null", - "object", - ], - }, - "type": [ - "null", - "array", - ], - }, - "weight": { - "type": [ - "null", - "integer", - ], - }, - }, - "type": "object", - }, - "name": "another_stream", - "sourceDefinedPrimaryKey": [], - "supportedSyncModes": [ - "full_refresh", - ], - }, - }, - ], - }, - "transformations": [], -} -`; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts deleted file mode 100644 index 5c6c5863c60..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts +++ /dev/null @@ -1,982 +0,0 @@ -import { SyncSchema, SyncSchemaStream } from "core/domain/catalog"; -import { - DestinationSyncMode, - FieldTransformTransformType, - StreamDescriptor, - StreamTransformTransformType, - SyncMode, -} from "core/request/AirbyteClient"; - -import calculateInitialCatalog from "./calculateInitialCatalog"; - -const mockSyncSchemaStream: SyncSchemaStream = { - id: "1", - stream: { - sourceDefinedCursor: true, - defaultCursorField: ["source_cursor"], - sourceDefinedPrimaryKey: [["new_primary_key"]], - jsonSchema: {}, - name: "test", - namespace: "namespace-test", - supportedSyncModes: [], - }, - config: { - destinationSyncMode: DestinationSyncMode.append, - selected: false, - syncMode: SyncMode.full_refresh, - cursorField: ["old_cursor"], - primaryKey: [["old_primary_key"]], - aliasName: "", - }, -}; - -describe("calculateInitialCatalog", () => { - it("should assign ids to all streams", () => { - const { id, ...restProps } = mockSyncSchemaStream; - - const values = calculateInitialCatalog( - { - streams: [restProps], - } as unknown as SyncSchema, - [], - [], - false - ); - - values.streams.forEach((stream) => { - expect(stream).toHaveProperty("id", "0"); - }); - }); - - it("should set default 'FullRefresh' if 'supportedSyncModes' in stream is empty(or null)", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - - const values = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - supportedSyncModes: null, - }, - config, - }, - ], - } as unknown as SyncSchema, - [], - [], - false - ); - - values.streams.forEach((stream) => { - expect(stream).toHaveProperty("stream.supportedSyncModes", [SyncMode.full_refresh]); - }); - }); - - it("should not select Incremental | Append Dedup if no source defined primary key is available", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - - const values = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedCursor: true, - defaultCursorField: ["id"], - supportedSyncModes: [SyncMode.full_refresh, SyncMode.incremental], - sourceDefinedPrimaryKey: undefined, - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - syncMode: SyncMode.full_refresh, - }, - }, - { - id: "2", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedCursor: true, - defaultCursorField: ["updated_at"], - supportedSyncModes: [SyncMode.full_refresh, SyncMode.incremental], - sourceDefinedPrimaryKey: [], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.full_refresh, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup, DestinationSyncMode.overwrite], - [], - false - ); - - values.streams.forEach((stream) => { - expect(stream).toHaveProperty("config.syncMode", SyncMode.full_refresh); - expect(stream).toHaveProperty("config.destinationSyncMode", DestinationSyncMode.overwrite); - }); - }); - - it("should select 'Incremental(cursor defined) => Append Dedup'", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - - const values = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedCursor: true, - defaultCursorField: ["id"], - supportedSyncModes: [SyncMode.full_refresh, SyncMode.incremental], - sourceDefinedPrimaryKey: [ - ["primary", "field1"], - ["primary", "field2"], - ], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - syncMode: SyncMode.full_refresh, - }, - }, - { - id: "2", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedCursor: true, - defaultCursorField: ["updated_at"], - supportedSyncModes: [SyncMode.full_refresh, SyncMode.incremental], - sourceDefinedPrimaryKey: [ - ["primary", "field1"], - ["primary", "field2"], - ], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.full_refresh, - }, - }, - { - id: "3", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedCursor: true, - defaultCursorField: ["name"], - supportedSyncModes: [SyncMode.incremental], - sourceDefinedPrimaryKey: [ - ["primary", "field1"], - ["primary", "field2"], - ], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append, - syncMode: SyncMode.full_refresh, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [], - false - ); - - values.streams.forEach((stream) => { - expect(stream).toHaveProperty("config.syncMode", SyncMode.incremental); - expect(stream).toHaveProperty("config.destinationSyncMode", DestinationSyncMode.append_dedup); - }); - }); - - it("should select 'Full Refresh => Overwrite'", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - - const values = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - syncMode: SyncMode.incremental, - }, - }, - { - id: "2", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.full_refresh, - }, - }, - { - id: "3", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append, - syncMode: SyncMode.full_refresh, - }, - }, - ], - }, - [DestinationSyncMode.overwrite], - [], - false - ); - - values.streams.forEach((stream) => { - expect(stream).toHaveProperty("config.syncMode", SyncMode.full_refresh); - expect(stream).toHaveProperty("config.destinationSyncMode", DestinationSyncMode.overwrite); - }); - }); - - it("should select 'Incremental => Append'", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - - const values = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - syncMode: SyncMode.incremental, - }, - }, - { - id: "2", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.full_refresh, - }, - }, - { - id: "3", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append, - syncMode: SyncMode.full_refresh, - }, - }, - ], - }, - [DestinationSyncMode.append], - [], - false - ); - - values.streams.forEach((stream) => { - expect(stream).toHaveProperty("config.syncMode", SyncMode.incremental); - expect(stream).toHaveProperty("config.destinationSyncMode", DestinationSyncMode.append); - }); - }); - - it("should select 'Full Refresh => Append'", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - - const values = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [SyncMode.full_refresh], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - syncMode: SyncMode.incremental, - }, - }, - { - id: "2", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [SyncMode.full_refresh], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - }, - }, - { - id: "3", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [SyncMode.full_refresh], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append, - syncMode: SyncMode.incremental, - }, - }, - ], - }, - [DestinationSyncMode.append], - [], - false - ); - - values.streams.forEach((stream) => { - expect(stream).toHaveProperty("config.syncMode", SyncMode.full_refresh); - expect(stream).toHaveProperty("config.destinationSyncMode", DestinationSyncMode.append); - }); - }); - - it("should not change syncMode, destinationSyncMode in EditMode", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - - const { streams: calculatedStreams } = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - supportedSyncModes: [], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append, - syncMode: SyncMode.full_refresh, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [], - true - ); - - expect(calculatedStreams[0]).toHaveProperty("stream.supportedSyncModes", [SyncMode.full_refresh]); - - expect(calculatedStreams[0]).toHaveProperty("config.syncMode", SyncMode.full_refresh); - expect(calculatedStreams[0]).toHaveProperty("config.destinationSyncMode", DestinationSyncMode.append); - }); - - it("should set the default cursorField value when it's available and no cursorField is selected", () => { - const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; - - const { streams: calculatedStreams } = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - defaultCursorField: ["default_path"], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - cursorField: [], - syncMode: SyncMode.full_refresh, - }, - }, - { - id: "2", - stream: { - ...sourceDefinedStream, - name: "test", - defaultCursorField: ["default_path"], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - cursorField: ["selected_path"], - syncMode: SyncMode.full_refresh, - }, - }, - { - id: "3", - stream: { - ...sourceDefinedStream, - name: "test", - defaultCursorField: [], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - cursorField: [], - syncMode: SyncMode.full_refresh, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [], - false - ); - - expect(calculatedStreams[0]).toHaveProperty("config.cursorField", ["default_path"]); - expect(calculatedStreams[1]).toHaveProperty("config.cursorField", ["selected_path"]); - expect(calculatedStreams[2]).toHaveProperty("config.cursorField", []); - }); - - it("source defined properties should override the saved properties", () => { - const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; - - const { streams: calculatedStreams } = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [], - true - ); - // primary keys - expect(calculatedStreams[0].stream?.sourceDefinedPrimaryKey).toEqual(sourceDefinedStream?.sourceDefinedPrimaryKey); - expect(calculatedStreams[0].config?.primaryKey).toEqual(calculatedStreams[0].stream?.sourceDefinedPrimaryKey); - - // cursor field - expect(calculatedStreams[0].stream?.sourceDefinedCursor).toBeTruthy(); - expect(calculatedStreams[0].stream?.defaultCursorField).toEqual(sourceDefinedStream?.defaultCursorField); - expect(calculatedStreams[0].config?.cursorField).toEqual(calculatedStreams[0].stream?.defaultCursorField); - }); - - it("should keep original configured primary key if no source-defined primary key", () => { - const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; - - const { streams: calculatedStreams } = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedCursor: undefined, - defaultCursorField: [], - sourceDefinedPrimaryKey: [], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - cursorField: [], - syncMode: SyncMode.full_refresh, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [], - false - ); - - expect(calculatedStreams[0].config?.primaryKey).toEqual(config?.primaryKey); - }); - - it("should not override config cursor if sourceDefinedCursor is false", () => { - const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; - - const { streams: calculatedStreams } = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedCursor: false, - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [], - false - ); - - expect(calculatedStreams[0].config?.cursorField).toEqual(config?.cursorField); - }); - - it("should keep its original config if source-defined primary key matches config primary key", () => { - const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; - - const { streams: calculatedStreams } = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedPrimaryKey: [["old_primary_key"]], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [], - false - ); - - expect(calculatedStreams[0].config?.primaryKey).toEqual(calculatedStreams[0].stream?.sourceDefinedPrimaryKey); - }); - - it("should not change primary key or cursor if isEditMode is false", () => { - const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; - - const { streams: calculatedStreams } = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [], - false - ); - // primary keys - expect(calculatedStreams[0].config?.primaryKey).toEqual(config?.primaryKey); - - // cursor field - expect(calculatedStreams[0].config?.cursorField).toEqual(config?.cursorField); - }); - - it("should calculate optimal sync mode if stream is new", () => { - const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; - - const newStreamDescriptors: StreamDescriptor[] = [{ name: "test", namespace: "namespace-test" }]; - - const { streams: calculatedStreams } = calculateInitialCatalog( - { - streams: [ - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - namespace: "namespace-test", - sourceDefinedCursor: true, - defaultCursorField: ["id"], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - syncMode: SyncMode.incremental, - }, - }, - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test2", - namespace: "namespace-test", - sourceDefinedCursor: true, - defaultCursorField: ["id"], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.overwrite, - syncMode: SyncMode.incremental, - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [], - true, - newStreamDescriptors - ); - - // new stream has its sync mode calculated - expect(calculatedStreams[0].config?.syncMode).toEqual(SyncMode.incremental); - expect(calculatedStreams[0].config?.destinationSyncMode).toEqual(DestinationSyncMode.append_dedup); - - // existing stream remains as-is - expect(calculatedStreams[1].config?.syncMode).toEqual(SyncMode.incremental); - expect(calculatedStreams[1].config?.destinationSyncMode).toEqual(DestinationSyncMode.overwrite); - }); - - it("should remove the entire primary key if any path from it was removed", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - const values = calculateInitialCatalog( - { - streams: [ - // Stream with breaking change - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedPrimaryKey: [], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - primaryKey: [["id"], ["email"]], - }, - }, - // Should not be affected - { - id: "2", - stream: { - ...sourceDefinedStream, - name: "test-2", - sourceDefinedPrimaryKey: [], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - primaryKey: [["id"]], - }, - }, - // Has change, but the source-defined primary key will fix it - { - id: "3", - stream: { - ...sourceDefinedStream, - name: "test-3", - sourceDefinedPrimaryKey: [["accountId"], ["userId"]], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - primaryKey: [["id"]], - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [ - { - transformType: StreamTransformTransformType.update_stream, - streamDescriptor: { name: "test", namespace: "namespace-test" }, - updateStream: [ - { - breaking: true, - transformType: FieldTransformTransformType.remove_field, - fieldName: ["id"], - }, - ], - }, - { - transformType: StreamTransformTransformType.update_stream, - streamDescriptor: { name: "test-3", namespace: "namespace-test" }, - updateStream: [ - { - breaking: false, - transformType: FieldTransformTransformType.add_field, - fieldName: ["userId"], - }, - ], - }, - ], - true - ); - expect(values.streams[0].config?.primaryKey).toEqual([]); // was entirely cleared - expect(values.streams[1].config?.primaryKey).toEqual([["id"]]); // was not affected - expect(values.streams[2].config?.primaryKey).toEqual([["accountId"], ["userId"]]); // was not affected because it's source-defined - }); - - it("should remove cursor from config if the old cursor field was removed, even if there is a default", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - const values = calculateInitialCatalog( - { - streams: [ - // With breaking change - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedCursor: false, - defaultCursorField: ["id"], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - cursorField: ["updated_at"], - }, - }, - // Will be unaffected - { - id: "2", - stream: { - ...sourceDefinedStream, - name: "test-2", - sourceDefinedCursor: true, - defaultCursorField: ["updated_at"], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - cursorField: ["updated_at"], - primaryKey: [["id"]], - }, - }, - // Has breaking change but the updated stream source-defined cursor will fix it - { - id: "3", - stream: { - ...sourceDefinedStream, - name: "test-3", - sourceDefinedCursor: true, - defaultCursorField: ["created_at"], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - cursorField: ["updated_at"], - primaryKey: [["id"]], - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [ - { - transformType: StreamTransformTransformType.update_stream, - streamDescriptor: { name: "test", namespace: "namespace-test" }, - updateStream: [ - { - breaking: true, - transformType: FieldTransformTransformType.remove_field, - fieldName: ["updated_at"], - }, - ], - }, - { - transformType: StreamTransformTransformType.update_stream, - streamDescriptor: { name: "test-3", namespace: "namespace-test" }, - updateStream: [ - { - breaking: true, - transformType: FieldTransformTransformType.remove_field, - fieldName: ["updated_at"], - }, - { - breaking: false, - transformType: FieldTransformTransformType.add_field, - fieldName: ["created_at"], - }, - ], - }, - ], - true - ); - expect(values.streams[0].config?.cursorField).toEqual([]); // was entirely cleared and not replaced with default - expect(values.streams[1].config?.cursorField).toEqual(["updated_at"]); // was unaffected - expect(values.streams[2].config?.cursorField).toEqual(["created_at"]); // was unaffected - }); - - it("should clear multiple config fields if multiple fields were removed", () => { - const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; - const values = calculateInitialCatalog( - { - streams: [ - // Breaking. Should be cleared - { - id: "1", - stream: { - ...sourceDefinedStream, - name: "test", - sourceDefinedCursor: false, - defaultCursorField: ["id"], - sourceDefinedPrimaryKey: [], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - cursorField: ["updated_at"], - primaryKey: [["primary_key"], ["another_field"]], - }, - }, - // Should be unaffected - { - id: "2", - stream: { - ...sourceDefinedStream, - name: "test-2", - sourceDefinedCursor: true, - defaultCursorField: ["updated_at"], - sourceDefinedPrimaryKey: [], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - cursorField: ["updated_at"], - primaryKey: [["id"]], - }, - }, - // Should stay unaffected because updated stream will assign new source-defined cursor - { - id: "3", - stream: { - ...sourceDefinedStream, - name: "test-3", - sourceDefinedCursor: true, - defaultCursorField: ["created_at"], - sourceDefinedPrimaryKey: [], - supportedSyncModes: [SyncMode.incremental], - }, - config: { - ...config, - destinationSyncMode: DestinationSyncMode.append_dedup, - syncMode: SyncMode.incremental, - cursorField: ["updated_at"], - primaryKey: [["id"]], - }, - }, - ], - }, - [DestinationSyncMode.append_dedup], - [ - { - transformType: StreamTransformTransformType.update_stream, - streamDescriptor: { name: "test", namespace: "namespace-test" }, - updateStream: [ - { - breaking: true, - transformType: FieldTransformTransformType.remove_field, - fieldName: ["updated_at"], - }, - { - breaking: true, - transformType: FieldTransformTransformType.remove_field, - fieldName: ["primary_key"], - }, - ], - }, - { - transformType: StreamTransformTransformType.update_stream, - streamDescriptor: { name: "test-3", namespace: "namespace-test" }, - updateStream: [ - { - breaking: true, - transformType: FieldTransformTransformType.remove_field, - fieldName: ["updated_at"], - }, - { - breaking: false, - transformType: FieldTransformTransformType.add_field, - fieldName: ["created_at"], - }, - { - breaking: true, - transformType: FieldTransformTransformType.remove_field, - fieldName: ["primary_key"], - }, - ], - }, - ], - true - ); - - expect(values.streams[0].config?.primaryKey).toEqual([]); // was entirely cleared and not replaced with default - expect(values.streams[0].config?.cursorField).toEqual([]); // was entirely cleared and not replaced with default - - expect(values.streams[1].config?.primaryKey).toEqual([["id"]]); // was unaffected - expect(values.streams[1].config?.cursorField).toEqual(["updated_at"]); // was unaffected - - expect(values.streams[2].config?.primaryKey).toEqual([["id"]]); // was unaffected - expect(values.streams[2].config?.cursorField).toEqual(["created_at"]); // was unaffected because it's a source-defined cursor - }); -}); diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts deleted file mode 100644 index 2a642487bfa..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts +++ /dev/null @@ -1,220 +0,0 @@ -import isEqual from "lodash/isEqual"; - -import { SyncSchema, SyncSchemaStream } from "core/domain/catalog"; -import { - DestinationSyncMode, - SyncMode, - AirbyteStreamConfiguration, - StreamDescriptor, - StreamTransform, -} from "core/request/AirbyteClient"; - -const getDefaultCursorField = (streamNode: SyncSchemaStream): string[] => { - if (streamNode.stream?.defaultCursorField?.length) { - return streamNode.stream.defaultCursorField; - } - return streamNode.config?.cursorField || []; -}; - -const clearBreakingFieldChanges = (nodeStream: SyncSchemaStream, breakingChangesByStream: StreamTransform[]) => { - if (!breakingChangesByStream.length || !nodeStream.config) { - return nodeStream; - } - - const { primaryKey, cursorField } = nodeStream.config; - - let clearPrimaryKey = false; - let clearCursorField = false; - - for (const streamTransformation of breakingChangesByStream) { - if (!streamTransformation.updateStream || !streamTransformation.updateStream?.length) { - continue; - } - - // get all of the removed field paths for this transformation - const breakingFieldPaths = streamTransformation.updateStream - .filter(({ breaking }) => breaking) - .map((update) => update.fieldName); - - // if there is a primary key in the config, and any of its field paths were removed, we'll be clearing it - if ( - !!primaryKey?.length && - primaryKey?.some((primaryKeyPath) => breakingFieldPaths.some((path) => isEqual(primaryKeyPath, path))) - ) { - clearPrimaryKey = true; - } - - // if there is a cursor field, and any of its field path was removed, we'll be clearing it - if (!!cursorField?.length && breakingFieldPaths.some((path) => isEqual(path, cursorField))) { - clearCursorField = true; - } - } - - if (clearPrimaryKey || clearCursorField) { - return { - ...nodeStream, - config: { - ...nodeStream.config, - primaryKey: clearPrimaryKey ? [] : nodeStream.config.primaryKey, - cursorField: clearCursorField ? [] : nodeStream.config.cursorField, - }, - }; - } - - return nodeStream; -}; - -const verifySourceDefinedProperties = (streamNode: SyncSchemaStream, isEditMode: boolean) => { - if (!streamNode.stream || !streamNode.config || !isEditMode) { - return streamNode; - } - - const { - stream: { sourceDefinedPrimaryKey, sourceDefinedCursor }, - } = streamNode; - - // if there's a source-defined cursor and the mode is correct, set the config to the default - if (sourceDefinedCursor) { - streamNode.config.cursorField = streamNode.stream.defaultCursorField; - } - - // if the primary key doesn't need to be calculated from the source, just return the node - if (!sourceDefinedPrimaryKey || sourceDefinedPrimaryKey.length === 0) { - return streamNode; - } - - // override the primary key with what the source said - streamNode.config.primaryKey = sourceDefinedPrimaryKey; - - return streamNode; -}; - -const verifySupportedSyncModes = (streamNode: SyncSchemaStream): SyncSchemaStream => { - if (!streamNode.stream) { - return streamNode; - } - const { - stream: { supportedSyncModes }, - } = streamNode; - - if (supportedSyncModes?.length) { - return streamNode; - } - return { ...streamNode, stream: { ...streamNode.stream, supportedSyncModes: [SyncMode.full_refresh] } }; -}; - -const verifyConfigCursorField = (streamNode: SyncSchemaStream): SyncSchemaStream => { - if (!streamNode.config) { - return streamNode; - } - const { config } = streamNode; - - return { - ...streamNode, - config: { - ...config, - cursorField: config.cursorField?.length ? config.cursorField : getDefaultCursorField(streamNode), - }, - }; -}; - -const getOptimalSyncMode = ( - streamNode: SyncSchemaStream, - supportedDestinationSyncModes: DestinationSyncMode[] -): SyncSchemaStream => { - const updateStreamConfig = ( - config: Pick - ): SyncSchemaStream => ({ - ...streamNode, - config: { ...streamNode.config, ...config }, - }); - if (!streamNode.stream?.supportedSyncModes) { - return streamNode; - } - const { - stream: { supportedSyncModes, sourceDefinedCursor, sourceDefinedPrimaryKey }, - } = streamNode; - - if ( - supportedSyncModes.includes(SyncMode.incremental) && - supportedDestinationSyncModes.includes(DestinationSyncMode.append_dedup) && - sourceDefinedCursor && - sourceDefinedPrimaryKey?.length - ) { - return updateStreamConfig({ - syncMode: SyncMode.incremental, - destinationSyncMode: DestinationSyncMode.append_dedup, - }); - } - - if (supportedDestinationSyncModes.includes(DestinationSyncMode.overwrite)) { - return updateStreamConfig({ - syncMode: SyncMode.full_refresh, - destinationSyncMode: DestinationSyncMode.overwrite, - }); - } - - if ( - supportedSyncModes.includes(SyncMode.incremental) && - supportedDestinationSyncModes.includes(DestinationSyncMode.append) - ) { - return updateStreamConfig({ - syncMode: SyncMode.incremental, - destinationSyncMode: DestinationSyncMode.append, - }); - } - - if (supportedDestinationSyncModes.includes(DestinationSyncMode.append)) { - return updateStreamConfig({ - syncMode: SyncMode.full_refresh, - destinationSyncMode: DestinationSyncMode.append, - }); - } - return streamNode; -}; - -/** - * @deprecated will be removed during clean up - https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - * @see calculateInitialCatalogHookForm.ts - */ -const calculateInitialCatalog = ( - schema: SyncSchema, - supportedDestinationSyncModes: DestinationSyncMode[], - streamsWithBreakingFieldChanges?: StreamTransform[], - isNotCreateMode?: boolean, - newStreamDescriptors?: StreamDescriptor[] -): SyncSchema => { - return { - streams: schema.streams.map((apiNode, id) => { - const nodeWithId: SyncSchemaStream = { ...apiNode, id: id.toString() }; - const nodeStream = verifySourceDefinedProperties(verifySupportedSyncModes(nodeWithId), isNotCreateMode || false); - - // if the stream is new since a refresh, verify cursor and get optimal sync modes - const isStreamNew = newStreamDescriptors?.some( - (streamIdFromDiff) => - streamIdFromDiff.name === nodeStream.stream?.name && - streamIdFromDiff.namespace === nodeStream.stream?.namespace - ); - - // if we're in edit or readonly mode and the stream is not new, check for breaking changes then return - if (isNotCreateMode && !isStreamNew) { - // narrow down the breaking field changes from this connection to only those relevant to this stream - const breakingChangesByStream = - streamsWithBreakingFieldChanges && streamsWithBreakingFieldChanges.length > 0 - ? streamsWithBreakingFieldChanges.filter(({ streamDescriptor }) => { - return ( - streamDescriptor.name === nodeStream.stream?.name && - streamDescriptor.namespace === nodeStream.stream?.namespace - ); - }) - : []; - - return clearBreakingFieldChanges(nodeStream, breakingChangesByStream); - } - - return getOptimalSyncMode(verifyConfigCursorField(nodeStream), supportedDestinationSyncModes); - }), - }; -}; - -export default calculateInitialCatalog; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.test.ts b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.test.ts deleted file mode 100644 index f34babb0e2a..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.test.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { renderHook } from "@testing-library/react"; - -import { mockConnection } from "test-utils/mock-data/mockConnection"; -import { - mockDestinationDefinitionSpecification, - mockDestinationDefinitionVersion, -} from "test-utils/mock-data/mockDestination"; -import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; -import { TestWrapper as wrapper } from "test-utils/testutils"; - -import { NormalizationType } from "area/connection/types"; -import { ConnectionScheduleTimeUnit, OperationRead } from "core/request/AirbyteClient"; - -import { mapFormPropsToOperation, useFrequencyDropdownData, useInitialValues } from "./formConfig"; -import { frequencyConfig } from "./frequencyConfig"; - -jest.mock("core/api", () => ({ - useCurrentWorkspace: () => mockWorkspace, -})); - -describe("#useFrequencyDropdownData", () => { - it("should return only default frequencies when no additional frequency is provided", () => { - const { result } = renderHook(() => useFrequencyDropdownData(undefined), { wrapper }); - expect(result.current.map((item) => item.value)).toEqual(["manual", "cron", ...frequencyConfig]); - }); - - it("should return only default frequencies when additional frequency is already present", () => { - const additionalFrequency = { - basicSchedule: { - units: 1, - timeUnit: ConnectionScheduleTimeUnit.hours, - }, - }; - const { result } = renderHook(() => useFrequencyDropdownData(additionalFrequency), { wrapper }); - expect(result.current.map((item) => item.value)).toEqual(["manual", "cron", ...frequencyConfig]); - }); - - it("should include additional frequency when provided and unique", () => { - const additionalFrequency = { - basicSchedule: { - units: 7, - timeUnit: ConnectionScheduleTimeUnit.minutes, - }, - }; - const { result } = renderHook(() => useFrequencyDropdownData(additionalFrequency), { wrapper }); - - // +1 for additionalFrequency, +2 for cron and manual frequencies - expect(result.current.length).toEqual(frequencyConfig.length + 1 + 2); - expect(result.current).toContainEqual({ label: "Every 7 minutes", value: { units: 7, timeUnit: "minutes" } }); - }); -}); - -describe("#mapFormPropsToOperation", () => { - const workspaceId = "asdf"; - const normalization: OperationRead = { - workspaceId, - operationId: "asdf", - name: "asdf", - operatorConfiguration: { - operatorType: "normalization", - }, - }; - const dbtCloudJob: OperationRead = { - workspaceId, - operationId: "testDbtCloudJob", - name: "testDbtCloudJob", - operatorConfiguration: { - operatorType: "webhook", - }, - }; - - it("should add any included transformations", () => { - expect( - mapFormPropsToOperation( - { - transformations: [normalization, dbtCloudJob], - }, - undefined, - "asdf" - ) - ).toEqual([normalization, dbtCloudJob]); - }); - - it("should add a basic normalization if normalization is set to basic", () => { - expect( - mapFormPropsToOperation( - { - normalization: NormalizationType.basic, - }, - - undefined, - workspaceId - ) - ).toEqual([ - { - name: "Normalization", - operatorConfiguration: { - normalization: { - option: "basic", - }, - operatorType: "normalization", - }, - workspaceId, - }, - ]); - }); - - it("should include any provided initial operations and not include the basic normalization operation when normalization type is basic", () => { - expect( - mapFormPropsToOperation( - { - normalization: NormalizationType.basic, - }, - [normalization], - workspaceId - ) - ).toEqual([normalization]); - }); - - it("should only include webhook operations from initial operations and not include the basic normalization operation when normalization type is raw", () => { - expect( - mapFormPropsToOperation( - { - normalization: NormalizationType.raw, - }, - [normalization], - workspaceId - ) - ).toEqual([]); - - expect( - mapFormPropsToOperation( - { - normalization: NormalizationType.raw, - }, - [normalization, dbtCloudJob], - workspaceId - ) - ).toEqual([dbtCloudJob]); - }); - - it("should include provided transformations when normalization type is raw, but not any provided normalizations", () => { - expect( - mapFormPropsToOperation( - { - normalization: NormalizationType.raw, - transformations: [normalization], - }, - [normalization], - workspaceId - ) - ).toEqual([normalization]); - }); - - it("should include provided transformations and normalizations when normalization type is basic", () => { - expect( - mapFormPropsToOperation( - { - normalization: NormalizationType.basic, - transformations: [normalization], - }, - [normalization], - workspaceId - ) - ).toEqual([normalization, normalization]); - }); - - it("should include provided transformations and default normalization when normalization type is basic and no normalizations have been provided", () => { - expect( - mapFormPropsToOperation( - { - normalization: NormalizationType.basic, - transformations: [normalization], - }, - undefined, - workspaceId - ) - ).toEqual([ - { - name: "Normalization", - operatorConfiguration: { - normalization: { - option: "basic", - }, - operatorType: "normalization", - }, - workspaceId, - }, - normalization, - ]); - }); -}); - -describe("#useInitialValues", () => { - it("should generate initial values w/ no 'not create' mode", () => { - const { result } = renderHook(() => - useInitialValues(mockConnection, mockDestinationDefinitionVersion, mockDestinationDefinitionSpecification) - ); - expect(result.current).toMatchSnapshot(); - expect(result.current.name).toBeDefined(); - }); - - it("should generate initial values w/ 'not create' mode: false", () => { - const { result } = renderHook(() => - useInitialValues(mockConnection, mockDestinationDefinitionVersion, mockDestinationDefinitionSpecification, false) - ); - expect(result.current).toMatchSnapshot(); - expect(result.current.name).toBeDefined(); - }); - - it("should generate initial values w/ 'not create' mode: true", () => { - const { result } = renderHook(() => - useInitialValues(mockConnection, mockDestinationDefinitionVersion, mockDestinationDefinitionSpecification, true) - ); - expect(result.current).toMatchSnapshot(); - expect(result.current.name).toBeUndefined(); - }); - - // This is a low-priority test - it.todo( - "should test for supportsDbt+initialValues.transformations and supportsNormalization+initialValues.normalization" - ); -}); diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx deleted file mode 100644 index acfd23e8bfc..00000000000 --- a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx +++ /dev/null @@ -1,464 +0,0 @@ -import { useMemo } from "react"; -import { useIntl } from "react-intl"; -import * as yup from "yup"; - -import { DropDownOptionDataItem } from "components/ui/DropDown"; - -import { NormalizationType } from "area/connection/types"; -import { validateCronExpression, validateCronFrequencyOneHourOrMore } from "area/connection/utils"; -import { isDbtTransformation, isNormalizationTransformation, isWebhookTransformation } from "area/connection/utils"; -import { ConnectionValues, useCurrentWorkspace } from "core/api"; -import { - ActorDefinitionVersionRead, - ConnectionScheduleData, - ConnectionScheduleType, - DestinationDefinitionSpecificationRead, - DestinationSyncMode, - Geography, - NamespaceDefinitionType, - NonBreakingChangesPreference, - OperationCreate, - OperationRead, - OperatorType, - SchemaChange, - SyncMode, - WebBackendConnectionRead, -} from "core/api/types/AirbyteClient"; -import { SyncSchema } from "core/domain/catalog"; -import { SOURCE_NAMESPACE_TAG } from "core/domain/connector/source"; -import { FeatureItem, useFeature } from "core/services/features"; -import { ConnectionFormMode, ConnectionOrPartialConnection } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useExperiment } from "hooks/services/Experiment"; - -import calculateInitialCatalog from "./calculateInitialCatalog"; -import { frequencyConfig } from "./frequencyConfig"; -import { DbtOperationRead } from "../TransformationHookForm"; - -export interface FormikConnectionFormValues { - name?: string; - scheduleType?: ConnectionScheduleType | null; - scheduleData?: ConnectionScheduleData | null; - nonBreakingChangesPreference?: NonBreakingChangesPreference | null; - prefix: string; - syncCatalog: SyncSchema; - namespaceDefinition?: NamespaceDefinitionType; - namespaceFormat: string; - transformations?: OperationRead[]; - normalization?: NormalizationType; - geography: Geography; -} - -export type ConnectionFormValues = ConnectionValues; - -export const SUPPORTED_MODES: Array<[SyncMode, DestinationSyncMode]> = [ - [SyncMode.incremental, DestinationSyncMode.append_dedup], - [SyncMode.full_refresh, DestinationSyncMode.overwrite], - [SyncMode.incremental, DestinationSyncMode.append], - [SyncMode.full_refresh, DestinationSyncMode.append], -]; - -const DEFAULT_SCHEDULE: ConnectionScheduleData = { - basicSchedule: { - units: 24, - timeUnit: "hours", - }, -}; - -export function useDefaultTransformation(): OperationCreate { - const workspace = useCurrentWorkspace(); - return { - name: "My dbt transformations", - workspaceId: workspace.workspaceId, - operatorConfiguration: { - operatorType: OperatorType.dbt, - dbt: { - gitRepoUrl: "", // TODO: Does this need a value? - dockerImage: "fishtownanalytics/dbt:1.0.0", - dbtArguments: "run", - }, - }, - }; -} - -const createConnectionValidationSchema = ( - mode: ConnectionFormMode, - allowSubOneHourCronExpressions: boolean, - allowAutoDetectSchema: boolean -) => { - return yup - .object({ - // The connection name during Editing is handled separately from the form - name: mode === "create" ? yup.string().required("form.empty.error") : yup.string().notRequired(), - geography: yup.mixed().oneOf(Object.values(Geography)), - scheduleType: yup - .string() - .oneOf([ConnectionScheduleType.manual, ConnectionScheduleType.basic, ConnectionScheduleType.cron]), - scheduleData: yup.mixed().when("scheduleType", (scheduleType) => { - if (scheduleType === ConnectionScheduleType.basic) { - return yup.object({ - basicSchedule: yup - .object({ - units: yup.number().required("form.empty.error"), - timeUnit: yup.string().required("form.empty.error"), - }) - .defined("form.empty.error"), - }); - } else if (scheduleType === ConnectionScheduleType.manual) { - return yup.mixed().notRequired(); - } - return yup.object({ - cron: yup - .object({ - cronExpression: yup - .string() - .trim() - .required("form.empty.error") - .test("validCron", (value, { createError, path }) => { - const validation = validateCronExpression(value); - return validation.isValid === true - ? true - : createError({ - path, - message: validation.message ?? "form.cronExpression.invalid", - }); - }) - .test( - "validCronFrequency", - "form.cronExpression.underOneHourNotAllowed", - (expression) => allowSubOneHourCronExpressions || validateCronFrequencyOneHourOrMore(expression) - ), - cronTimeZone: yup.string().required("form.empty.error"), - }) - .defined("form.empty.error"), - }); - }), - nonBreakingChangesPreference: allowAutoDetectSchema - ? yup.mixed().oneOf(Object.values(NonBreakingChangesPreference)).required("form.empty.error") - : yup.mixed().notRequired(), - namespaceDefinition: yup - .string() - .oneOf([ - NamespaceDefinitionType.destination, - NamespaceDefinitionType.source, - NamespaceDefinitionType.customformat, - ]) - .required("form.empty.error"), - namespaceFormat: yup.string().when("namespaceDefinition", { - is: NamespaceDefinitionType.customformat, - then: yup.string().trim().required("form.empty.error"), - }), - prefix: yup.string(), - syncCatalog: yup.object({ - streams: yup - .array() - .of( - yup.object({ - id: yup - .string() - // This is required to get rid of id fields we are using to detect stream for edition - .when("$isRequest", (isRequest: boolean, schema: yup.StringSchema) => - isRequest ? schema.strip(true) : schema - ), - stream: yup.object(), - config: yup - .object({ - selected: yup.boolean(), - syncMode: yup.string(), - destinationSyncMode: yup.string(), - primaryKey: yup.array().of(yup.array().of(yup.string())), - cursorField: yup.array().of(yup.string()), - }) - .test({ - message: "form.empty.error", - test(value) { - if (!value.selected) { - return true; - } - - const errors: yup.ValidationError[] = []; - const pathRoot = "syncCatalog"; - - // it's possible that primaryKey array is always present - // however yup couldn't determine type correctly even with .required() call - if ( - DestinationSyncMode.append_dedup === value.destinationSyncMode && - value.primaryKey?.length === 0 - ) { - errors.push( - this.createError({ - message: "connectionForm.primaryKey.required", - path: `${pathRoot}.streams[${this.parent.id}].config.primaryKey`, - }) - ); - } - - // it's possible that cursorField array is always present - // however yup couldn't determine type correctly even with .required() call - if ( - SyncMode.incremental === value.syncMode && - !this.parent.stream.sourceDefinedCursor && - value.cursorField?.length === 0 - ) { - errors.push( - this.createError({ - message: "connectionForm.cursorField.required", - path: `${pathRoot}.streams[${this.parent.id}].config.cursorField`, - }) - ); - } - - return errors.length > 0 ? new yup.ValidationError(errors) : true; - }, - }), - }) - ) - .test( - "syncCatalog.streams.required", - "connectionForm.streams.required", - (streams) => streams?.some(({ config }) => !!config.selected) ?? false - ), - }), - }) - .noUnknown(); -}; - -interface CreateConnectionValidationSchemaArgs { - mode: ConnectionFormMode; -} - -export const useConnectionValidationSchema = ({ mode }: CreateConnectionValidationSchemaArgs) => { - const allowSubOneHourCronExpressions = useFeature(FeatureItem.AllowSyncSubOneHourCronExpressions); - const allowAutoDetectSchema = useFeature(FeatureItem.AllowAutoDetectSchema); - - return useMemo( - () => createConnectionValidationSchema(mode, allowSubOneHourCronExpressions, allowAutoDetectSchema), - [allowAutoDetectSchema, allowSubOneHourCronExpressions, mode] - ); -}; - -export type ConnectionValidationSchema = ReturnType; - -/** - * Returns {@link Operation}[] - * - * Maps UI representation of Transformation and Normalization - * into API's {@link Operation} representation. - * - * Always puts normalization as first operation - * @param values - * @param initialOperations - * @param workspaceId - */ -// TODO: need to split this mapper for each type of operation(transformations, normalizations, webhooks) -export function mapFormPropsToOperation( - values: { - transformations?: OperationRead[]; - normalization?: NormalizationType; - }, - initialOperations: OperationRead[] = [], - workspaceId: string -): OperationCreate[] { - const newOperations: OperationCreate[] = []; - - if (values.normalization && values.normalization !== NormalizationType.raw) { - const normalizationOperation = initialOperations.find(isNormalizationTransformation); - - if (normalizationOperation) { - newOperations.push(normalizationOperation); - } else { - newOperations.push({ - name: "Normalization", - workspaceId, - operatorConfiguration: { - operatorType: OperatorType.normalization, - normalization: { - option: values.normalization, - }, - }, - }); - } - } - - if (values.transformations) { - newOperations.push(...values.transformations); - } - - // webhook operations (e.g. dbt Cloud jobs in the Airbyte Cloud integration) are managed - // by separate sub-forms; they should not be ignored (which would cause accidental - // deletions), but managing them should not be combined with this (already-confusing) - // codepath, either. - newOperations.push(...initialOperations.filter(isWebhookTransformation)); - - return newOperations; -} - -/** - * get transformation operations only - * @param operations - */ -export const getInitialTransformations = (operations: OperationRead[]): DbtOperationRead[] => - operations?.filter(isDbtTransformation) ?? []; - -export const getInitialNormalization = ( - operations?: Array, - isNotCreateMode?: boolean -): NormalizationType => { - const initialNormalization = - operations?.find(isNormalizationTransformation)?.operatorConfiguration?.normalization?.option; - - return initialNormalization - ? NormalizationType[initialNormalization] - : isNotCreateMode - ? NormalizationType.raw - : NormalizationType.basic; -}; - -export const useInitialValues = ( - connection: ConnectionOrPartialConnection, - destDefinitionVersion: ActorDefinitionVersionRead, - destDefinitionSpecification: DestinationDefinitionSpecificationRead, - isNotCreateMode?: boolean -): FormikConnectionFormValues => { - const autoPropagationEnabled = useExperiment("autopropagation.enabled", false); - const workspace = useCurrentWorkspace(); - const { catalogDiff } = connection; - - const defaultNonBreakingChangesPreference = autoPropagationEnabled - ? NonBreakingChangesPreference.propagate_columns - : NonBreakingChangesPreference.ignore; - - // used to determine if we should calculate optimal sync mode - const newStreamDescriptors = catalogDiff?.transforms - .filter((transform) => transform.transformType === "add_stream") - .map((stream) => stream.streamDescriptor); - - // used to determine if we need to clear any primary keys or cursor fields that were removed - const streamTransformsWithBreakingChange = useMemo(() => { - if (connection.schemaChange === SchemaChange.breaking) { - return catalogDiff?.transforms.filter((streamTransform) => { - if (streamTransform.transformType === "update_stream") { - return streamTransform.updateStream?.filter((fieldTransform) => fieldTransform.breaking === true); - } - return false; - }); - } - return undefined; - }, [catalogDiff?.transforms, connection]); - - const initialSchema = useMemo( - () => - calculateInitialCatalog( - connection.syncCatalog, - destDefinitionSpecification?.supportedDestinationSyncModes || [], - streamTransformsWithBreakingChange, - isNotCreateMode, - newStreamDescriptors - ), - [ - streamTransformsWithBreakingChange, - connection.syncCatalog, - destDefinitionSpecification?.supportedDestinationSyncModes, - isNotCreateMode, - newStreamDescriptors, - ] - ); - - return useMemo(() => { - const initialValues: FormikConnectionFormValues = { - syncCatalog: initialSchema, - scheduleType: connection.connectionId ? connection.scheduleType : ConnectionScheduleType.basic, - scheduleData: connection.connectionId ? connection.scheduleData ?? null : DEFAULT_SCHEDULE, - nonBreakingChangesPreference: connection.nonBreakingChangesPreference ?? defaultNonBreakingChangesPreference, - prefix: connection.prefix || "", - namespaceDefinition: connection.namespaceDefinition || NamespaceDefinitionType.destination, - namespaceFormat: connection.namespaceFormat ?? SOURCE_NAMESPACE_TAG, - geography: connection.geography || workspace.defaultGeography || "auto", - }; - - // Is Create Mode - if (!isNotCreateMode) { - initialValues.name = connection.name ?? `${connection.source.name} → ${connection.destination.name}`; - } - - const operations = connection.operations ?? []; - - if (destDefinitionVersion.supportsDbt) { - initialValues.transformations = getInitialTransformations(operations); - } - - if (destDefinitionVersion.normalizationConfig?.supported) { - initialValues.normalization = getInitialNormalization(operations, isNotCreateMode); - } - - return initialValues; - }, [ - connection.connectionId, - connection.destination.name, - connection.geography, - connection.name, - connection.namespaceDefinition, - connection.namespaceFormat, - connection.nonBreakingChangesPreference, - connection.operations, - connection.prefix, - connection.scheduleData, - connection.scheduleType, - connection.source.name, - defaultNonBreakingChangesPreference, - destDefinitionVersion.supportsDbt, - destDefinitionVersion.normalizationConfig, - initialSchema, - isNotCreateMode, - workspace, - ]); -}; - -export const useFrequencyDropdownData = ( - additionalFrequency: WebBackendConnectionRead["scheduleData"] -): DropDownOptionDataItem[] => { - const { formatMessage } = useIntl(); - - return useMemo(() => { - const frequencies = [...frequencyConfig]; - if (additionalFrequency?.basicSchedule) { - const additionalFreqAlreadyPresent = frequencies.some( - (frequency) => - frequency?.timeUnit === additionalFrequency.basicSchedule?.timeUnit && - frequency?.units === additionalFrequency.basicSchedule?.units - ); - if (!additionalFreqAlreadyPresent) { - frequencies.push(additionalFrequency.basicSchedule); - } - } - - const basicFrequencies = frequencies.map((frequency) => ({ - value: frequency, - label: formatMessage( - { - id: `form.every.${frequency.timeUnit}`, - }, - { value: frequency.units } - ), - })); - - // Add Manual and Custom to the frequencies list - const customFrequency = formatMessage({ - id: "frequency.cron", - }); - const manualFrequency = formatMessage({ - id: "frequency.manual", - }); - const otherFrequencies = [ - { - label: manualFrequency, - value: manualFrequency.toLowerCase(), - }, - { - label: customFrequency, - value: customFrequency.toLowerCase(), - }, - ]; - - return [...otherFrequencies, ...basicFrequencies]; - }, [formatMessage, additionalFrequency]); -}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/hookFormConfig.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/hookFormConfig.tsx index 92656134a02..086f445e1cf 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/hookFormConfig.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/hookFormConfig.tsx @@ -2,8 +2,16 @@ import { useMemo } from "react"; import { FieldArrayWithId } from "react-hook-form"; import { NormalizationType } from "area/connection/types"; +import { isDbtTransformation, isNormalizationTransformation } from "area/connection/utils"; import { useCurrentWorkspace } from "core/api"; -import { AirbyteCatalog, DestinationDefinitionSpecificationRead, SchemaChange } from "core/api/types/AirbyteClient"; +import { + AirbyteCatalog, + DestinationDefinitionSpecificationRead, + DestinationSyncMode, + OperationCreate, + SchemaChange, + SyncMode, +} from "core/api/types/AirbyteClient"; import { ActorDefinitionVersionRead, ConnectionScheduleData, @@ -14,14 +22,16 @@ import { OperationRead, } from "core/request/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; -import { ConnectionOrPartialConnection } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; +import { + ConnectionOrPartialConnection, + useConnectionFormService, +} from "hooks/services/ConnectionForm/ConnectionFormService"; import { useExperiment } from "hooks/services/Experiment"; import { calculateInitialCatalogHookForm } from "./calculateInitialCatalogHookForm"; -import { getInitialNormalization, getInitialTransformations } from "./formConfig"; import { BASIC_FREQUENCY_DEFAULT_VALUE } from "./ScheduleHookFormField/useBasicFrequencyDropdownDataHookForm"; import { createConnectionValidationSchema } from "./schema"; +import { DbtOperationRead } from "../TransformationHookForm"; /** * react-hook-form form values type for the connection form @@ -46,13 +56,23 @@ export interface HookFormConnectionFormValues { */ export type SyncStreamFieldWithId = FieldArrayWithId; +/** + * supported sync modes for the sync catalog row + */ +export const SUPPORTED_MODES: Array<[SyncMode, DestinationSyncMode]> = [ + [SyncMode.incremental, DestinationSyncMode.append_dedup], + [SyncMode.full_refresh, DestinationSyncMode.overwrite], + [SyncMode.incremental, DestinationSyncMode.append], + [SyncMode.full_refresh, DestinationSyncMode.append], +]; + /** * useConnectionValidationSchema with additional arguments */ export const useConnectionHookFormValidationSchema = () => { const allowSubOneHourCronExpressions = useFeature(FeatureItem.AllowSyncSubOneHourCronExpressions); const allowAutoDetectSchema = useFeature(FeatureItem.AllowAutoDetectSchema); - const { mode } = useConnectionHookFormService(); + const { mode } = useConnectionFormService(); return useMemo( () => createConnectionValidationSchema(mode, allowSubOneHourCronExpressions, allowAutoDetectSchema), @@ -60,12 +80,38 @@ export const useConnectionHookFormValidationSchema = () => { ); }; +/** + * get transformation operations only + * @param operations + */ +export const getInitialTransformations = (operations: OperationRead[]): DbtOperationRead[] => + operations?.filter(isDbtTransformation) ?? []; + +/** + * get normalization initial normalization type + * @param operations + * @param isEditMode + */ +export const getInitialNormalization = ( + operations?: Array, + isEditMode?: boolean +): NormalizationType => { + const initialNormalization = + operations?.find(isNormalizationTransformation)?.operatorConfiguration?.normalization?.option; + + return initialNormalization + ? NormalizationType[initialNormalization] + : isEditMode + ? NormalizationType.raw + : NormalizationType.basic; +}; + // react-hook-form form values type for the connection form. export const useInitialHookFormValues = ( connection: ConnectionOrPartialConnection, destDefinitionVersion: ActorDefinitionVersionRead, destDefinitionSpecification: DestinationDefinitionSpecificationRead, - isNotCreateMode?: boolean + isEditMode?: boolean ): HookFormConnectionFormValues => { const autoPropagationEnabled = useExperiment("autopropagation.enabled", false); const workspace = useCurrentWorkspace(); @@ -99,14 +145,14 @@ export const useInitialHookFormValues = ( connection.syncCatalog, destDefinitionSpecification?.supportedDestinationSyncModes || [], streamTransformsWithBreakingChange, - isNotCreateMode, + isEditMode, newStreamDescriptors ), [ connection.syncCatalog, destDefinitionSpecification?.supportedDestinationSyncModes, streamTransformsWithBreakingChange, - isNotCreateMode, + isEditMode, newStreamDescriptors, ] ); @@ -114,7 +160,7 @@ export const useInitialHookFormValues = ( return useMemo(() => { const initialValues: HookFormConnectionFormValues = { // set name field - ...(isNotCreateMode + ...(isEditMode ? {} : { name: connection.name ?? `${connection.source.name} → ${connection.destination.name}`, @@ -147,7 +193,7 @@ export const useInitialHookFormValues = ( geography: connection.geography || workspace.defaultGeography || "auto", ...{ ...(destDefinitionVersion.supportsDbt && { - normalization: getInitialNormalization(connection.operations ?? [], isNotCreateMode), + normalization: getInitialNormalization(connection.operations ?? [], isEditMode), }), }, ...{ @@ -174,7 +220,7 @@ export const useInitialHookFormValues = ( defaultNonBreakingChangesPreference, workspace.defaultGeography, destDefinitionVersion.supportsDbt, - isNotCreateMode, + isEditMode, initialSchema, ]); }; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/refreshSourceSchemaWithConfirmationOnDirty.ts b/airbyte-webapp/src/components/connection/ConnectionForm/refreshSourceSchemaWithConfirmationOnDirty.ts index 5b7df39924f..1d57f8ec964 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/refreshSourceSchemaWithConfirmationOnDirty.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/refreshSourceSchemaWithConfirmationOnDirty.ts @@ -5,9 +5,9 @@ import { useConnectionFormService } from "hooks/services/ConnectionForm/Connecti import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; export const useRefreshSourceSchemaWithConfirmationOnDirty = (dirty: boolean) => { - const { clearFormChange } = useFormChangeTrackerService(); + const { clearAllFormChanges } = useFormChangeTrackerService(); const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); - const { formId, refreshSchema } = useConnectionFormService(); + const { refreshSchema } = useConnectionFormService(); return useCallback(() => { if (dirty) { @@ -17,12 +17,12 @@ export const useRefreshSourceSchemaWithConfirmationOnDirty = (dirty: boolean) => submitButtonText: "connection.updateSchema.formChanged.confirm", onSubmit: () => { closeConfirmationModal(); - clearFormChange(formId); + clearAllFormChanges(); refreshSchema(); }, }); } else { refreshSchema(); } - }, [clearFormChange, closeConfirmationModal, dirty, formId, openConfirmationModal, refreshSchema]); + }, [clearAllFormChanges, closeConfirmationModal, dirty, openConfirmationModal, refreshSchema]); }; diff --git a/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.module.scss b/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.module.scss index 20f47b0d255..cb51beadf88 100644 --- a/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.module.scss +++ b/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.module.scss @@ -82,8 +82,8 @@ } .moreIcon { - width: 28px; - height: 28px; + width: 38px; + height: 38px; color: colors.$blue; } diff --git a/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.tsx b/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.tsx index a55c490bb89..7928b7da058 100644 --- a/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.tsx @@ -1,27 +1,23 @@ -import { faCircleQuestion } from "@fortawesome/free-regular-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import { useMemo, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; +import { Icon } from "components/ui/Icon"; import { Link } from "components/ui/Link"; import { Text } from "components/ui/Text"; import { Tooltip } from "components/ui/Tooltip"; import { ConnectorIds, SvgIcon } from "area/connector/utils"; -import { useCurrentWorkspace } from "core/api"; +import { useCurrentWorkspace, useSourceDefinitionList, useDestinationDefinitionList } from "core/api"; import { DestinationDefinitionRead, SourceDefinitionRead } from "core/request/AirbyteClient"; import { links } from "core/utils/links"; import { useExperiment } from "hooks/services/Experiment"; import { ConnectionRoutePaths, DestinationPaths, RoutePaths } from "pages/routePaths"; -import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; -import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; import { AirbyteIllustration, HighlightIndex } from "./AirbyteIllustration"; import styles from "./ConnectionOnboarding.module.scss"; -import PlusIcon from "./plusIcon.svg?react"; import { SOURCE_DEFINITION_PARAM } from "../CreateConnection/CreateNewSource"; import { NEW_SOURCE_TYPE, SOURCE_TYPE_PARAM } from "../CreateConnection/SelectSource"; @@ -136,7 +132,7 @@ export const ConnectionOnboarding: React.FC = () => { - + } > @@ -184,7 +180,7 @@ export const ConnectionOnboarding: React.FC = () => { alignItems="center" justifyContent="center" > - + } @@ -204,8 +200,7 @@ export const ConnectionOnboarding: React.FC = () => { - {" "} - + } > @@ -259,7 +254,7 @@ export const ConnectionOnboarding: React.FC = () => { alignItems="center" justifyContent="center" > - + } diff --git a/airbyte-webapp/src/components/connection/ConnectionStatusIndicator/ConnectionStatusIndicator.tsx b/airbyte-webapp/src/components/connection/ConnectionStatusIndicator/ConnectionStatusIndicator.tsx index 3a7ea823cbb..87565563a78 100644 --- a/airbyte-webapp/src/components/connection/ConnectionStatusIndicator/ConnectionStatusIndicator.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionStatusIndicator/ConnectionStatusIndicator.tsx @@ -1,10 +1,6 @@ import classNames from "classnames"; import React from "react"; -import { ClockIcon } from "components/icons/ClockIcon"; -import { SimpleCircleIcon } from "components/icons/SimpleCircleIcon"; -import { SuccessIcon } from "components/icons/SuccessIcon"; -import { WarningCircleIcon } from "components/icons/WarningCircleIcon"; import { Icon } from "components/ui/Icon"; import { LoadingSpinner } from "components/ui/LoadingSpinner"; @@ -21,13 +17,13 @@ export enum ConnectionStatusIndicatorStatus { } const ICON_BY_STATUS: Readonly> = { - onTime: , - onTrack: , - error: , - disabled: , - pending: , - late: , - actionRequired: , + onTime: , + onTrack: , + error: , + disabled: , + pending: , + late: , + actionRequired: , }; const STYLE_BY_STATUS: Readonly> = { diff --git a/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncButtons.tsx b/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncButtons.tsx index b916a30ef13..62b49c43ab5 100644 --- a/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncButtons.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncButtons.tsx @@ -1,11 +1,9 @@ -import { faEllipsisV } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { useCallback } from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { RotateIcon } from "components/icons/RotateIcon"; import { Button, ButtonVariant } from "components/ui/Button"; import { DropdownMenu, DropdownMenuOptionType } from "components/ui/DropdownMenu"; +import { Icon } from "components/ui/Icon"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; @@ -69,7 +67,7 @@ export const ConnectionSyncButtons: React.FC = ({ {!jobSyncRunning && !jobResetRunning && (
); diff --git a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx index d612ab4cc39..feedb394302 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx @@ -10,10 +10,10 @@ import { Button } from "components/ui/Button"; import { Icon } from "components/ui/Icon"; import { useSuggestedDestinations } from "area/connector/utils"; +import { useDestinationDefinitionList } from "core/api"; import { AppActionCodes, useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { useCreateDestination } from "hooks/services/useDestinationHook"; -import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; import { DESTINATION_ID_PARAM, DESTINATION_TYPE_PARAM } from "./SelectDestination"; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx index fee36209156..f301054199d 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx @@ -8,11 +8,11 @@ import { Button } from "components/ui/Button"; import { Icon } from "components/ui/Icon"; import { useSuggestedSources } from "area/connector/utils"; +import { useSourceDefinitionList } from "core/api"; import { AppActionCodes, useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { useCreateSource } from "hooks/services/useSourceHook"; import { SourceForm, SourceFormValues } from "pages/source/CreateSourcePage/SourceForm"; -import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; import { SOURCE_ID_PARAM, SOURCE_TYPE_PARAM } from "./SelectSource"; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx b/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx index 88ce58e8874..58f85871d0c 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx @@ -9,8 +9,8 @@ import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; +import { useSourceDefinitionList } from "core/api"; import { useSourceList } from "hooks/services/useSourceHook"; -import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; import { CreateNewSource, SOURCE_DEFINITION_PARAM } from "./CreateNewSource"; import { RadioButtonTiles } from "./RadioButtonTiles"; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.module.scss b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.module.scss deleted file mode 100644 index f83a1535d70..00000000000 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.module.scss +++ /dev/null @@ -1,12 +0,0 @@ -@use "scss/variables"; - -.connectionFormContainer { - width: 100%; - min-width: variables.$min-width-wide-table-container; - - > form { - display: flex; - flex-direction: column; - gap: variables.$spacing-md; - } -} diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx deleted file mode 100644 index e75580a11c7..00000000000 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx +++ /dev/null @@ -1,161 +0,0 @@ -import { Form, Formik, FormikHelpers } from "formik"; -import React, { Suspense, useCallback, useEffect, useState } from "react"; -import { useNavigate } from "react-router-dom"; - -import { ConnectionFormFields } from "components/connection/ConnectionForm/ConnectionFormFields"; -import { CreateControls } from "components/connection/ConnectionForm/CreateControls"; -import { - FormikConnectionFormValues, - useConnectionValidationSchema, -} from "components/connection/ConnectionForm/formConfig"; -import { OperationsSection } from "components/connection/ConnectionForm/OperationsSection"; -import LoadingSchema from "components/LoadingSchema"; - -import { useGetDestinationFromSearchParams, useGetSourceFromSearchParams } from "area/connector/utils"; -import { useCurrentWorkspaceId } from "area/workspace/utils"; -import { useCreateConnection } from "core/api"; -import { FeatureItem, useFeature } from "core/services/features"; -import { - ConnectionFormServiceProvider, - tidyConnectionFormValues, - useConnectionFormService, -} from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useExperimentContext } from "hooks/services/Experiment"; -import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; -import { SchemaError as SchemaErrorType, useDiscoverSchema } from "hooks/services/useSourceHook"; - -import styles from "./CreateConnectionForm.module.scss"; -import { CreateConnectionNameField } from "./CreateConnectionNameField"; -import { DataResidency } from "./DataResidency"; -import { SchemaError } from "./SchemaError"; -import { useAnalyticsTrackFunctions } from "./useAnalyticsTrackFunctions"; - -interface CreateConnectionPropsInner { - schemaError: SchemaErrorType; -} - -const CreateConnectionFormInner: React.FC = ({ schemaError }) => { - const navigate = useNavigate(); - const canEditDataGeographies = useFeature(FeatureItem.AllowChangeDataGeographies); - const { mutateAsync: createConnection } = useCreateConnection(); - const { clearFormChange } = useFormChangeTrackerService(); - - const workspaceId = useCurrentWorkspaceId(); - - const { connection, initialValues, mode, formId, getErrorMessage, setSubmitError } = useConnectionFormService(); - const [editingTransformation, setEditingTransformation] = useState(false); - const validationSchema = useConnectionValidationSchema({ mode }); - useExperimentContext("source-definition", connection.source?.sourceDefinitionId); - - const onFormSubmit = useCallback( - async (formValues: FormikConnectionFormValues, formikHelpers: FormikHelpers) => { - const values = tidyConnectionFormValues(formValues, workspaceId, validationSchema); - - try { - const createdConnection = await createConnection({ - values, - source: connection.source, - destination: connection.destination, - sourceDefinition: { - sourceDefinitionId: connection.source?.sourceDefinitionId ?? "", - }, - destinationDefinition: { - name: connection.destination?.name ?? "", - destinationDefinitionId: connection.destination?.destinationDefinitionId ?? "", - }, - sourceCatalogId: connection.catalogId, - }); - - formikHelpers.resetForm(); - // We need to clear the form changes otherwise the dirty form intercept service will prevent navigation - clearFormChange(formId); - - navigate(`../../connections/${createdConnection.connectionId}`); - } catch (e) { - setSubmitError(e); - } - }, - [ - workspaceId, - validationSchema, - createConnection, - connection.source, - connection.destination, - connection.catalogId, - clearFormChange, - formId, - navigate, - setSubmitError, - ] - ); - - if (schemaError) { - return ; - } - - return ( - }> -
- - {({ isSubmitting, isValid, dirty, errors, validateForm }) => ( -
- - {canEditDataGeographies && } - - setEditingTransformation(true)} - onEndEditTransformation={() => setEditingTransformation(false)} - /> - - - )} - -
- - ); -}; - -export const CreateConnectionForm: React.FC = () => { - const source = useGetSourceFromSearchParams(); - const destination = useGetDestinationFromSearchParams(); - const { trackFailure } = useAnalyticsTrackFunctions(); - - const { schema, isLoading, schemaErrorStatus, catalogId, onDiscoverSchema } = useDiscoverSchema( - source.sourceId, - true - ); - - useEffect(() => { - if (schemaErrorStatus) { - trackFailure(source, destination, schemaErrorStatus); - } - // we need to track the schemaErrorStatus changes only - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [schemaErrorStatus]); - - if (!schema) { - return ; - } - - const partialConnection = { - syncCatalog: schema, - destination, - source, - catalogId, - }; - - return ( - - {isLoading ? : } - - ); -}; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.module.scss b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.module.scss deleted file mode 100644 index 778ee0f7334..00000000000 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.module.scss +++ /dev/null @@ -1 +0,0 @@ -@forward "../ConnectionForm/ConnectionFormFields.module.scss"; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.tsx deleted file mode 100644 index 515b6a08b61..00000000000 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import { Field, FieldProps } from "formik"; -import { FormattedMessage, useIntl } from "react-intl"; - -import { Section } from "components/connection/ConnectionForm/Section"; -import { ControlLabels } from "components/LabeledControl"; -import { Input } from "components/ui/Input"; - -import { FormFieldLayout } from "../ConnectionForm/FormFieldLayout"; - -/** - * @deprecated it's formik version of CreateConnectionNameField form control and will be removed in the future, use ConnectionNameHookFormCard instead - * @see ConnectionNameHookFormCard - * @constructor - */ -export const CreateConnectionNameField = () => { - const { formatMessage } = useIntl(); - - return ( -
}> - - {({ field, meta, form }: FieldProps) => ( - - } - infoTooltipContent={formatMessage({ - id: "form.connectionName.message", - })} - /> - - - )} - -
- ); -}; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidency.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidency.tsx deleted file mode 100644 index e9a63c76b40..00000000000 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidency.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import { Field, FieldProps, useFormikContext } from "formik"; -import React from "react"; -import { FormattedMessage, useIntl } from "react-intl"; - -import { DataGeographyDropdown } from "components/common/DataGeographyDropdown"; -import { Section } from "components/connection/ConnectionForm/Section"; -import { ControlLabels } from "components/LabeledControl"; - -import { useAvailableGeographies } from "core/api"; -import { Geography } from "core/request/AirbyteClient"; -import { links } from "core/utils/links"; - -import { FormFieldLayout } from "../ConnectionForm/FormFieldLayout"; - -interface DataResidencyProps { - name?: string; -} - -/** - * @deprecated it's formik version of DataResidency form control and will be removed in the future, use DataResidencyHookForm instead - * @see DataResidencyHookFormCard - */ -export const DataResidency: React.FC = ({ name = "geography" }) => { - const { formatMessage } = useIntl(); - const { setFieldValue } = useFormikContext(); - const { geographies } = useAvailableGeographies(); - - return ( -
- - {({ field, form }: FieldProps) => ( - - } - infoTooltipContent={ - ( - - {node} - - ), - docLink: (node: React.ReactNode) => ( - - {node} - - ), - }} - /> - } - /> - setFieldValue(name, geography)} - /> - - )} - -
- ); -}; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx index 4381920411c..a9a607896fc 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx @@ -1,11 +1,10 @@ -import { faRefresh } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { FormattedMessage } from "react-intl"; import { JobFailure } from "components/JobFailure"; import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { LogsRequestError } from "core/request/LogsRequestError"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; @@ -21,7 +20,7 @@ export const SchemaError = ({ schemaError }: { schemaError: Exclude - diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap deleted file mode 100644 index 4a3bdd2a37e..00000000000 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap +++ /dev/null @@ -1,1441 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`CreateConnectionForm should render 1`] = ` - -
-
-
-
-
-

- Connection -

-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-

- Configuration -

-
-
-
-
-
-
-
-
- - -
-
-
-
-
- Every 24 hours -
-
-
- -
-
- -
-
- -
-
-
-
-
-
-
-
-
-
-

- Destination default -

-
- -
-
-
-
-
-
-
-
-
-
-

- Mirror source name -

-
- -
-
-
-
-
-
-
-
-
- - -
-
-
-
-
- Ignore -
-
-
- -
-
- -
-
- -
-
-
-
-
-
-
-

- Activate the streams you want to sync -

- -
-
-
-
- -
-
-
- - -
-
-
-
-
-
- -

- Sync -

-
-
-

- Data destination - -

-
-
-

- Stream - -

-
-
-

- Sync mode - - - - - - - - - -

-
-
-

-

-
-

- Fields -

-
-
-
-
-
-
-
-
-
- -
-
-
- -

- '<destination schema> -

- -
-
- -

- another_stream -

- -
-
-
- -
-
-
-
-
-
-
-

- All -

-
-
-
-
-
-
-
-
- -
-
-
- -

- '<destination schema> -

- -
-
- -

- pokemon -

- -
-
-
- -
-
-
-
-
-
-
-

- All -

-
-
-
-
-
-
-
-
- -
-
-
- -

- '<destination schema> -

- -
-
- -

- pokemon2 -

- -
-
-
- -
-
-
-
-
-
-
-

- All -

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- Normalization & Transformation -

-
-
- - -
-
- - -
-
-
-
-
-

- No custom transformation -

- -
-
-
-
-
-
-
-
- -
-
- -
-
-
- -`; - -exports[`CreateConnectionForm should render when loading 1`] = ` - -
-
-
-
-
-
- Please wait a little bit more… -
-
-

- We are fetching the schema of your data source. -This should take less than a minute, but may take a few minutes on slow internet connections or data sources with a large amount of tables. -

-
-
-
-
- -`; - -exports[`CreateConnectionForm should render with an error 1`] = ` - -
-
-
-
- -
-
-
- -
-
- - Test Error - -
-
-
-
-
-
- -`; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/index.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/index.tsx deleted file mode 100644 index 8d57b01b25f..00000000000 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export * from "./CreateConnectionForm"; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.test.tsx similarity index 89% rename from airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx rename to airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.test.tsx index db7d790d168..add46b720a9 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.test.tsx @@ -2,7 +2,6 @@ import { act, render as tlr } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React from "react"; -import selectEvent from "react-select-event"; import { VirtuosoMockContext } from "react-virtuoso"; import { mockConnection } from "test-utils/mock-data/mockConnection"; @@ -22,15 +21,7 @@ import { TestWrapper, useMockIntersectionObserver } from "test-utils/testutils"; import { defaultOssFeatures, FeatureItem } from "core/services/features"; import * as sourceHook from "hooks/services/useSourceHook"; -import { CreateConnectionForm } from "./CreateConnectionForm"; - -jest.mock("services/connector/SourceDefinitionService", () => ({ - useSourceDefinition: () => mockSourceDefinition, -})); - -jest.mock("services/connector/DestinationDefinitionService", () => ({ - useDestinationDefinition: () => mockDestinationDefinition, -})); +import { CreateConnectionHookForm } from "./CreateConnectionHookForm"; jest.mock("area/workspace/utils", () => ({ useCurrentWorkspaceId: () => "workspace-id", @@ -44,6 +35,8 @@ jest.mock("core/api", () => ({ useDestinationDefinitionVersion: () => mockDestinationDefinitionVersion, useGetSourceDefinitionSpecification: () => mockSourceDefinitionSpecification, useGetDestinationDefinitionSpecification: () => mockDestinationDefinitionSpecification, + useSourceDefinition: () => mockSourceDefinition, + useDestinationDefinition: () => mockDestinationDefinition, })); jest.mock("area/connector/utils", () => ({ @@ -58,7 +51,7 @@ jest.mock("hooks/theme/useAirbyteTheme", () => ({ jest.setTimeout(40000); -describe("CreateConnectionForm", () => { +describe("CreateConnectionHookForm", () => { const Wrapper: React.FC> = ({ children }) => ( @@ -72,7 +65,7 @@ describe("CreateConnectionForm", () => { await act(async () => { renderResult = tlr( - + ); }); @@ -126,11 +119,12 @@ describe("CreateConnectionForm", () => { const container = tlr( - + ); - await selectEvent.select(container.getByTestId("scheduleData"), /cron/i); + await userEvent.click(container.getByTestId("schedule-type-listbox-button")); + await userEvent.click(container.getByTestId("cron-option")); const cronExpressionInput = container.getByTestId("cronExpression"); @@ -147,11 +141,12 @@ describe("CreateConnectionForm", () => { const container = tlr( - + ); - await selectEvent.select(container.getByTestId("scheduleData"), /cron/i); + await userEvent.click(container.getByTestId("schedule-type-listbox-button")); + await userEvent.click(container.getByTestId("cron-option")); const cronExpressionField = container.getByTestId("cronExpression"); @@ -170,11 +165,12 @@ describe("CreateConnectionForm", () => { const container = tlr( - + ); - await selectEvent.select(container.getByTestId("scheduleData"), /cron/i); + await userEvent.click(container.getByTestId("schedule-type-listbox-button")); + await userEvent.click(container.getByTestId("cron-option")); const cronExpressionField = container.getByTestId("cronExpression"); diff --git a/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.tsx b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.tsx index b7348e12d2d..c0f99673584 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.tsx @@ -10,9 +10,9 @@ import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useCreateConnection } from "core/api"; import { FeatureItem, useFeature } from "core/services/features"; import { - ConnectionHookFormServiceProvider, - useConnectionHookFormService, -} from "hooks/services/ConnectionForm/ConnectionHookFormService"; + ConnectionFormServiceProvider, + useConnectionFormService, +} from "hooks/services/ConnectionForm/ConnectionFormService"; import { useExperimentContext } from "hooks/services/Experiment"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { useDiscoverSchema } from "hooks/services/useSourceHook"; @@ -34,7 +34,7 @@ const CreateConnectionFormInner: React.FC = () => { const workspaceId = useCurrentWorkspaceId(); const { clearAllFormChanges } = useFormChangeTrackerService(); const { mutateAsync: createConnection } = useCreateConnection(); - const { connection, initialValues, setSubmitError } = useConnectionHookFormService(); + const { connection, initialValues, setSubmitError } = useConnectionFormService(); const canEditDataGeographies = useFeature(FeatureItem.AllowChangeDataGeographies); useExperimentContext("source-definition", connection.source?.sourceDefinitionId); @@ -135,7 +135,7 @@ export const CreateConnectionHookForm: React.FC = () => { }; return ( - { ) : ( )} - + ); }; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionHookForm/__snapshots__/CreateConnectionHookForm.test.tsx.snap b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/__snapshots__/CreateConnectionHookForm.test.tsx.snap new file mode 100644 index 00000000000..1b16599620b --- /dev/null +++ b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/__snapshots__/CreateConnectionHookForm.test.tsx.snap @@ -0,0 +1,1306 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CreateConnectionHookForm should render 1`] = ` + +
+
+
+
+
+
+

+ Connection +

+
+
+
+
+
+
+
+

+ Configuration +

+
+
+
+
+ +
+
+ +`; + +exports[`CreateConnectionHookForm should render when loading 1`] = ` + +
+
+
+
+
+
+ Please wait a little bit more… +
+
+

+ We are fetching the schema of your data source. +This should take less than a minute, but may take a few minutes on slow internet connections or data sources with a large amount of tables. +

+
+
+
+
+ +`; + +exports[`CreateConnectionHookForm should render with an error 1`] = ` + +
+
+
+
+ +
+
+
+
+
+
+ + Test Error + +
+
+
+
+
+
+ +`; diff --git a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx index f584a5679ec..9df252c6561 100644 --- a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx +++ b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx @@ -13,7 +13,6 @@ import { NamespaceDefinitionType } from "core/request/AirbyteClient"; import { DestinationNamespaceDescription } from "./DestinationNamespaceDescription"; import styles from "./DestinationNamespaceModal.module.scss"; -import { FormikConnectionFormValues } from "../ConnectionForm/formConfig"; import { HookFormConnectionFormValues } from "../ConnectionForm/hookFormConfig"; import { LabeledRadioButtonFormControl } from "../ConnectionForm/LabeledRadioButtonFormControl"; import { namespaceDefinitionSchema, namespaceFormatSchema } from "../ConnectionForm/schema"; @@ -52,15 +51,7 @@ const destinationNamespaceValidationSchema = yup.object().shape({ }); interface DestinationNamespaceModalProps { - /** - * temporary extend this interface since we use modal in Formik and react-hook-form forms - *TODO: remove FormikConnectionFormValues after successful CreateConnectionForm migration - *https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - */ - initialValues: Pick< - FormikConnectionFormValues | HookFormConnectionFormValues, - "namespaceDefinition" | "namespaceFormat" - >; + initialValues: Pick; onCloseModal: () => void; onSubmit: (values: DestinationNamespaceFormValues) => void; } diff --git a/airbyte-webapp/src/components/connection/DestinationStreamNamesModal/DestinationStreamNamesModal.tsx b/airbyte-webapp/src/components/connection/DestinationStreamNamesModal/DestinationStreamNamesModal.tsx index d69f006dae0..477513647d4 100644 --- a/airbyte-webapp/src/components/connection/DestinationStreamNamesModal/DestinationStreamNamesModal.tsx +++ b/airbyte-webapp/src/components/connection/DestinationStreamNamesModal/DestinationStreamNamesModal.tsx @@ -3,7 +3,6 @@ import { useFormContext } from "react-hook-form"; import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; -import { FormikConnectionFormValues } from "components/connection/ConnectionForm/formConfig"; import { Form, FormControl } from "components/forms"; import { ModalFormSubmissionButtons } from "components/forms/ModalFormSubmissionButtons"; import { Box } from "components/ui/Box"; @@ -55,17 +54,16 @@ const destinationStreamNamesValidationSchema = yup.object().shape({ .required("form.empty.error"), prefix: yup.string().when("streamNameDefinition", { is: StreamNameDefinitionValueType.Prefix, - then: yup.string().trim().required("form.empty.error"), + then: yup + .string() + .trim() + .required("form.empty.error") + .matches(/^[a-zA-Z0-9_]*$/, "form.invalidCharacters.alphanumericunder.error"), }), }); interface DestinationStreamNamesModalProps { - /** - * temporary extend this interface since we use modal in Formik and react-hook-form forms - *TODO: remove FormikConnectionFormValues after successful CreateConnectionForm migration - *https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - */ - initialValues: Pick; + initialValues: Pick; onCloseModal: () => void; onSubmit: (value: DestinationStreamNamesFormValues) => void; } diff --git a/airbyte-webapp/src/components/connection/EnabledControl/EnabledControl.tsx b/airbyte-webapp/src/components/connection/EnabledControl/EnabledControl.tsx index 785295541c3..f971dd48c73 100644 --- a/airbyte-webapp/src/components/connection/EnabledControl/EnabledControl.tsx +++ b/airbyte-webapp/src/components/connection/EnabledControl/EnabledControl.tsx @@ -6,9 +6,7 @@ import { useAsyncFn } from "react-use"; import { Switch } from "components/ui/Switch"; import { ConnectionStatus } from "core/request/AirbyteClient"; -import { getFrequencyFromScheduleData } from "core/services/analytics"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { getFrequencyFromScheduleData, Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import styles from "./EnabledControl.module.scss"; diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx b/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx index 93a4c258ee3..d05a03e937a 100644 --- a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx +++ b/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx @@ -1,12 +1,11 @@ -import { faDatabase, faDiagramNext } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import { FormattedMessage, useIntl } from "react-intl"; -import { JobWithAttempts } from "components/JobItem/types"; -import { getJobStatus } from "components/JobItem/utils"; +import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; +import { JobWithAttempts } from "area/connection/types/jobs"; +import { getJobStatus } from "area/connection/utils/jobs"; import { AttemptRead, AttemptStatus, SynchronousJobRead } from "core/request/AirbyteClient"; import { formatBytes } from "core/utils/numberHelper"; @@ -80,7 +79,7 @@ export const JobProgress: React.FC = ({ job, expanded }) => { {denominatorRecords > 0 && denominatorBytes > 0 && (
- + = ({ job, expanded }) => { /> - + > = { - [ConnectionStatusIndicatorStatus.ActionRequired]: , - [ConnectionStatusIndicatorStatus.Disabled]: , - [ConnectionStatusIndicatorStatus.Error]: , - [ConnectionStatusIndicatorStatus.Late]: , - [ConnectionStatusIndicatorStatus.Pending]: , - [ConnectionStatusIndicatorStatus.OnTime]: , - [ConnectionStatusIndicatorStatus.OnTrack]: , + [ConnectionStatusIndicatorStatus.ActionRequired]: , + [ConnectionStatusIndicatorStatus.Disabled]: , + [ConnectionStatusIndicatorStatus.Error]: , + [ConnectionStatusIndicatorStatus.Late]: , + [ConnectionStatusIndicatorStatus.Pending]: , + [ConnectionStatusIndicatorStatus.OnTime]: , + [ConnectionStatusIndicatorStatus.OnTrack]: , }; const STYLE_BY_STATUS: Readonly> = { diff --git a/airbyte-webapp/src/components/connection/TransformationForm/TransformationForm.module.scss b/airbyte-webapp/src/components/connection/TransformationForm/TransformationForm.module.scss deleted file mode 100644 index b471b8120d7..00000000000 --- a/airbyte-webapp/src/components/connection/TransformationForm/TransformationForm.module.scss +++ /dev/null @@ -1,22 +0,0 @@ -@use "../../../scss/variables"; - -.content { - display: flex; - flex-direction: row; -} - -.label { - margin-bottom: 20px; - - label { - width: calc(100% + #{variables.$spacing-xl}); - } -} - -.column { - width: 100%; - - &:first-child { - margin-right: variables.$spacing-lg; - } -} diff --git a/airbyte-webapp/src/components/connection/TransformationForm/TransformationForm.tsx b/airbyte-webapp/src/components/connection/TransformationForm/TransformationForm.tsx deleted file mode 100644 index d56802053dc..00000000000 --- a/airbyte-webapp/src/components/connection/TransformationForm/TransformationForm.tsx +++ /dev/null @@ -1,173 +0,0 @@ -import type { FormikErrors } from "formik/dist/types"; - -import { getIn, useFormik } from "formik"; -import React from "react"; -import { FormattedMessage, useIntl } from "react-intl"; - -import { FormChangeTracker } from "components/common/FormChangeTracker"; -import { ControlLabels } from "components/LabeledControl"; -import { Button } from "components/ui/Button"; -import { DropDown } from "components/ui/DropDown"; -import { Input } from "components/ui/Input"; -import { ModalBody, ModalFooter } from "components/ui/Modal"; - -import { useOperationsCheck } from "core/api"; -import { OperationCreate, OperationRead } from "core/request/AirbyteClient"; -import { links } from "core/utils/links"; -import { equal } from "core/utils/objects"; -import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; - -import styles from "./TransformationForm.module.scss"; -import { validationSchema } from "./utils"; - -interface TransformationProps { - transformation: OperationCreate; - onCancel: () => void; - onDone: (tr: OperationCreate) => void; - isNewTransformation?: boolean; -} - -function prepareLabelFields( - errors: FormikErrors, - name: string -): { error?: boolean; message?: React.ReactNode } { - const error = getIn(errors, name); - - const fields: { error?: boolean; message?: React.ReactNode } = {}; - - if (error) { - fields.error = true; - fields.message = ; - } - - return fields; -} - -// enum with only one value for the moment -const TransformationTypes = [{ value: "custom", label: "Custom DBT" }]; - -/** - * @deprecated it's Formik version of TransformationForm - * the new version is TransformationHookForm: - * @see TransformationHookForm - * @param transformation - * @param onCancel - * @param onDone - * @param isNewTransformation - * @constructor - */ -const TransformationForm: React.FC = ({ - transformation, - onCancel, - onDone, - isNewTransformation, -}) => { - const { formatMessage } = useIntl(); - const operationCheck = useOperationsCheck(); - const { clearFormChange } = useFormChangeTrackerService(); - const formId = useUniqueFormId(); - - const formik = useFormik({ - initialValues: transformation, - validationSchema, - onSubmit: async (values) => { - await operationCheck(values); - clearFormChange(formId); - onDone(values); - }, - }); - - const onFormCancel: React.MouseEventHandler = () => { - clearFormChange(formId); - onCancel?.(); - }; - - return ( - <> - - -
-
- } - > - - - - } - > - - - } - > - `<${node}>` } - )} - /> - -
- -
- }> - - - ( - - {node} - - ), - }} - /> - } - > - - - }> - - -
-
- - - - - - - ); -}; - -export default TransformationForm; diff --git a/airbyte-webapp/src/components/connection/TransformationForm/index.tsx b/airbyte-webapp/src/components/connection/TransformationForm/index.tsx deleted file mode 100644 index ecd29e245ef..00000000000 --- a/airbyte-webapp/src/components/connection/TransformationForm/index.tsx +++ /dev/null @@ -1,3 +0,0 @@ -import TransformationForm from "./TransformationForm"; - -export default TransformationForm; diff --git a/airbyte-webapp/src/components/connection/TransformationForm/utils.test.ts b/airbyte-webapp/src/components/connection/TransformationForm/utils.test.ts deleted file mode 100644 index 3da93a4e67d..00000000000 --- a/airbyte-webapp/src/components/connection/TransformationForm/utils.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import merge from "lodash/merge"; -import { InferType, ValidationError } from "yup"; - -import { validationSchema } from "./utils"; - -describe(" - validationSchema", () => { - const customTransformationFields: InferType = { - name: "test name", - operatorConfiguration: { - dbt: { - gitRepoUrl: "https://github.com/username/example.git", - dockerImage: "image", - dbtArguments: "arguments", - gitRepoBranch: "", - }, - }, - }; - - it("should successfully validate the schema", async () => { - const result = validationSchema.validate(customTransformationFields); - - await expect(result).resolves.toBeTruthy(); - }); - - it("should fail if 'name' is empty", async () => { - await expect(async () => { - await validationSchema.validateAt( - "name", - merge(customTransformationFields, { - name: "", - }) - ); - }).rejects.toThrow(ValidationError); - }); - - it("should fail if 'gitRepoUrl' is invalid", async () => { - await expect(async () => { - await validationSchema.validateAt( - "operatorConfiguration.dbt.gitRepoUrl", - merge(customTransformationFields, { - operatorConfiguration: { dbt: { gitRepoUrl: "" } }, - }) - ); - }).rejects.toThrow(ValidationError); - - await expect(async () => { - await validationSchema.validateAt( - "operatorConfiguration.dbt.gitRepoUrl", - merge(customTransformationFields, { - operatorConfiguration: { dbt: { gitRepoUrl: "https://github.com/username/example.git " } }, - }) - ); - }).rejects.toThrow(ValidationError); - - await expect(async () => { - await validationSchema.validateAt( - "operatorConfiguration.dbt.gitRepoUrl", - merge(customTransformationFields, { - operatorConfiguration: { dbt: { gitRepoUrl: "https://github.com/username/example.git/" } }, - }) - ); - }).rejects.toThrow(ValidationError); - }); - - it("should fail if 'dockerImage' is empty", async () => { - await expect(async () => { - await validationSchema.validateAt( - "operatorConfiguration.dbt.dockerImage", - merge(customTransformationFields, { - operatorConfiguration: { dbt: { dockerImage: "" } }, - }) - ); - }).rejects.toThrow(ValidationError); - }); - - it("should fail if 'dbtArguments' is empty", async () => { - await expect(async () => { - await validationSchema.validateAt( - "operatorConfiguration.dbt.dbtArguments", - merge(customTransformationFields, { - operatorConfiguration: { dbt: { dbtArguments: "" } }, - }) - ); - }).rejects.toThrow(ValidationError); - }); -}); diff --git a/airbyte-webapp/src/components/connection/TransformationForm/utils.ts b/airbyte-webapp/src/components/connection/TransformationForm/utils.ts deleted file mode 100644 index 50fa3879273..00000000000 --- a/airbyte-webapp/src/components/connection/TransformationForm/utils.ts +++ /dev/null @@ -1,16 +0,0 @@ -import * as yup from "yup"; - -export const validationSchema = yup.object({ - name: yup.string().required("form.empty.error"), - operatorConfiguration: yup.object({ - dbt: yup.object({ - gitRepoUrl: yup - .string() - .required("form.empty.error") - .matches(/((http(s)?)|(git@[\w.]+))(:(\/\/)?)([\w.@:/\-~]+)(\.git)$/, "form.repositoryUrl.invalidUrl"), - dockerImage: yup.string().required("form.empty.error"), - dbtArguments: yup.string().required("form.empty.error"), - gitRepoBranch: yup.string().nullable(), - }), - }), -}); diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.tsx index ee5beed5dab..e9251ec303b 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.tsx @@ -4,8 +4,8 @@ import isEqual from "lodash/isEqual"; import React, { useCallback, useMemo } from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { ArrowRightIcon } from "components/icons/ArrowRightIcon"; import { FlexContainer } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { Switch } from "components/ui/Switch"; import { Table } from "components/ui/Table"; import { TextWithOverflowTooltip } from "components/ui/Text"; @@ -273,11 +273,11 @@ export const StreamFieldsTable: React.FC = ({ }), columnHelper.group({ id: "arrow", - header: () => , + header: () => , columns: [ { id: "_", // leave the column name empty - cell: () => , + cell: () => , meta: { thClassName: styles.headerCell, tdClassName: styles.arrowCell, diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/SyncFieldCell.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/SyncFieldCell.tsx index fc785c59929..73e1b5d7cc6 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/SyncFieldCell.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/SyncFieldCell.tsx @@ -1,5 +1,4 @@ -import { useCallback } from "react"; -import React from "react"; +import React, { useCallback } from "react"; import { FormattedMessage } from "react-intl"; import { FlexContainer } from "components/ui/Flex"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeader.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeader.tsx deleted file mode 100644 index 5c28ecfdcf3..00000000000 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeader.tsx +++ /dev/null @@ -1,166 +0,0 @@ -import { faGear } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { setIn, useFormikContext } from "formik"; -import React from "react"; -import { FormattedMessage } from "react-intl"; - -import { FormikConnectionFormValues } from "components/connection/ConnectionForm/formConfig"; -import { Button } from "components/ui/Button"; -import { FlexContainer } from "components/ui/Flex"; -import { Switch } from "components/ui/Switch"; -import { Text } from "components/ui/Text"; -import { InfoTooltip, TooltipLearnMoreLink } from "components/ui/Tooltip"; - -import { SyncSchemaStream } from "core/domain/catalog"; -import { NamespaceDefinitionType } from "core/request/AirbyteClient"; -import { links } from "core/utils/links"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useModalService } from "hooks/services/Modal"; - -import styles from "./StreamsConfigTableHeader.module.scss"; -import { DestinationNamespaceModal, DestinationNamespaceFormValues } from "../../DestinationNamespaceModal"; -import { - DestinationStreamNamesModal, - DestinationStreamNamesFormValues, - StreamNameDefinitionValueType, -} from "../../DestinationStreamNamesModal"; -import { CellText, CellTextProps } from "../CellText"; - -const HeaderCell: React.FC> = ({ children, ...tableCellProps }) => ( - - - {children} - - -); - -interface StreamsConfigTableHeaderProps { - streams: SyncSchemaStream[]; - onStreamsChanged: (streams: SyncSchemaStream[]) => void; - syncSwitchDisabled?: boolean; -} - -export const StreamsConfigTableHeader: React.FC = ({ - streams, - onStreamsChanged, - syncSwitchDisabled, -}) => { - const { mode } = useConnectionFormService(); - const { openModal, closeModal } = useModalService(); - const formikProps = useFormikContext(); - - const destinationNamespaceHookFormChange = (value: DestinationNamespaceFormValues) => { - formikProps.setFieldValue("namespaceDefinition", value.namespaceDefinition); - - if (value.namespaceDefinition === NamespaceDefinitionType.customformat) { - formikProps.setFieldValue("namespaceFormat", value.namespaceFormat); - } - }; - - const destinationStreamNameHookFormChange = (value: DestinationStreamNamesFormValues) => { - formikProps.setFieldValue( - "prefix", - value.streamNameDefinition === StreamNameDefinitionValueType.Prefix ? value.prefix : "" - ); - }; - - const onToggleAllStreamsSyncSwitch = ({ target: { checked } }: React.ChangeEvent) => - onStreamsChanged( - streams.map((stream) => - setIn(stream, "config", { - ...stream.config, - selected: checked, - }) - ) - ); - const isPartOfStreamsSyncEnabled = () => - streams.some((stream) => stream.config?.selected) && - streams.filter((stream) => stream.config?.selected).length !== streams.length; - const areAllStreamsSyncEnabled = () => streams.every((stream) => stream.config?.selected) && streams.length > 0; - - return ( - - - - - - - - - - - - - - - - - - - - - - - - - - - - ); -}; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeaderHookForm.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeaderHookForm.tsx index e22538a9e67..511e73df1fc 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeaderHookForm.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeaderHookForm.tsx @@ -1,5 +1,3 @@ -import { faGear } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import set from "lodash/set"; import React from "react"; import { useFormContext } from "react-hook-form"; @@ -7,6 +5,7 @@ import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { Switch } from "components/ui/Switch"; import { Text } from "components/ui/Text"; import { InfoTooltip, TooltipLearnMoreLink } from "components/ui/Tooltip"; @@ -128,7 +127,7 @@ export const StreamsConfigTableHeaderHookForm: React.FC - + @@ -153,7 +152,7 @@ export const StreamsConfigTableHeaderHookForm: React.FC - + diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableRow.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableRow.tsx deleted file mode 100644 index 64257573c08..00000000000 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableRow.tsx +++ /dev/null @@ -1,244 +0,0 @@ -import classNames from "classnames"; -import React, { useMemo, useRef, useState } from "react"; -import { FormattedMessage } from "react-intl"; -import { useLocation, useNavigate } from "react-router-dom"; -import { useEffectOnce } from "react-use"; - -import { FlexContainer } from "components/ui/Flex"; -import { Switch } from "components/ui/Switch"; -import { Text, TextWithOverflowTooltip } from "components/ui/Text"; - -import { Path, SyncSchemaField, SyncSchemaStream } from "core/domain/catalog"; - -import { FieldSelectionStatus, FieldSelectionStatusVariant } from "./FieldSelectionStatus"; -import styles from "./StreamsConfigTableRow.module.scss"; -import { StreamsConfigTableRowStatus } from "./StreamsConfigTableRowStatus"; -import { useStreamsConfigTableRowProps } from "./useStreamsConfigTableRowProps"; -import { CellText } from "../CellText"; -import { LocationWithState } from "../SyncCatalog/SyncCatalogBody"; -import { SyncModeSelect, SyncModeValue } from "../SyncModeSelect"; -import { FieldPathType } from "../utils"; - -interface StreamsConfigTableRowProps { - stream: SyncSchemaStream; - destName: string; - destNamespace: string; - availableSyncModes: SyncModeValue[]; - onSelectSyncMode: (selectedMode: SyncModeValue) => void; - onSelectStream: () => void; - primitiveFields: SyncSchemaField[]; - pkType: FieldPathType; - onPrimaryKeyChange: (pkPath: Path[]) => void; - cursorType: FieldPathType; - onCursorChange: (cursorPath: Path) => void; - fields: SyncSchemaField[]; - openStreamDetailsPanel: () => void; - hasError: boolean; - configErrors?: Record; - disabled?: boolean; -} - -export const StreamsConfigTableRow: React.FC = ({ - stream, - destName, - destNamespace, - onSelectSyncMode, - onSelectStream, - availableSyncModes, - pkType, - cursorType, - fields, - openStreamDetailsPanel, - disabled, - configErrors, -}) => { - const { primaryKey, cursorField, syncMode, destinationSyncMode, selectedFields } = stream.config ?? {}; - - const pathDisplayName = (path: Path): string => path.join("."); - - const { defaultCursorField } = stream.stream ?? {}; - - const isCursorUndefined = useMemo(() => { - if (cursorType === "sourceDefined" && defaultCursorField?.length) { - return false; - } else if (cursorType === "required" && cursorField?.length) { - return false; - } - return true; - }, [cursorField?.length, cursorType, defaultCursorField?.length]); - - const isPrimaryKeyUndefined = useMemo(() => { - if (!primaryKey?.length) { - return true; - } - return false; - }, [primaryKey?.length]); - - const cursorFieldString = useMemo(() => { - if (cursorType === "sourceDefined") { - if (defaultCursorField?.length) { - return pathDisplayName(defaultCursorField); - } - return ; - } else if (cursorType === "required" && cursorField?.length) { - return pathDisplayName(cursorField); - } - return ; - }, [cursorType, cursorField, defaultCursorField]); - - const primaryKeyString = useMemo(() => { - if (!primaryKey?.length) { - return ; - } - return primaryKey.map(pathDisplayName).join(", "); - }, [primaryKey]); - - const syncSchema: SyncModeValue | undefined = useMemo(() => { - if (!syncMode || !destinationSyncMode) { - return undefined; - } - return { - syncMode, - destinationSyncMode, - }; - }, [syncMode, destinationSyncMode]); - - const fieldCount = fields?.length ?? 0; - const selectedFieldCount = selectedFields?.length ?? fieldCount; - const onRowClick: React.MouseEventHandler | undefined = - fieldCount > 0 - ? (e) => { - let target: Element | null = e.target as Element; - - // if the target is or has a *[data-noexpand] ancestor - // then exit, otherwise toggle expand - while (target) { - if (target.hasAttribute("data-noexpand")) { - return; - } - target = target.parentElement; - } - - openStreamDetailsPanel(); - } - : undefined; - - const { streamHeaderContentStyle, pillButtonVariant } = useStreamsConfigTableRowProps(stream); - - const { state: locationState, pathname } = useLocation() as LocationWithState; - const navigate = useNavigate(); - const rowRef = useRef(null); - const [highlighted, setHighlighted] = useState(false); - - useEffectOnce(() => { - let highlightTimeout: number; - let openTimeout: number; - - // Is it the stream we are looking for? - if (locationState?.streamName === stream.stream?.name && locationState?.namespace === stream.stream?.namespace) { - // Scroll to the stream and highlight it - if (locationState?.action === "showInReplicationTable" || locationState?.action === "openDetails") { - setHighlighted(true); - highlightTimeout = window.setTimeout(() => { - setHighlighted(false); - }, 1500); - } - - // Open the stream details - if (locationState?.action === "openDetails") { - openTimeout = window.setTimeout(() => { - rowRef.current?.click(); - }, 750); - } - // remove the redirection info from the location state - navigate(pathname, { replace: true }); - } - - return () => { - window.clearTimeout(highlightTimeout); - window.clearTimeout(openTimeout); - }; - }); - - return ( - - - - - - - - {destNamespace} - - - - {destName} - - - - - - {(cursorType || pkType) && ( - - {cursorType && ( - - - - - - {cursorFieldString} - - - )} - {pkType && ( - - - - - - {primaryKeyString} - - - )} - - )} - - - - - - ); -}; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableRowStatus.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableRowStatus.tsx index dbd0f80d124..662b998b6e4 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableRowStatus.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableRowStatus.tsx @@ -2,28 +2,26 @@ import classNames from "classnames"; import { useState } from "react"; import { useUpdateEffect } from "react-use"; -import { MinusIcon } from "components/icons/MinusIcon"; -import { ModificationIcon } from "components/icons/ModificationIcon"; -import { PlusIcon } from "components/icons/PlusIcon"; +import { Icon } from "components/ui/Icon"; -import { SyncSchemaStream } from "core/domain/catalog"; +import { AirbyteStreamAndConfiguration } from "core/api/types/AirbyteClient"; import styles from "./StreamsConfigTableRowStatus.module.scss"; import { StatusToDisplay, useStreamsConfigTableRowProps } from "./useStreamsConfigTableRowProps"; interface StreamsConfigTableRowStatusProps { - stream: SyncSchemaStream; + stream: AirbyteStreamAndConfiguration; className?: string; } const getIcon = (statusToDisplay: StatusToDisplay): React.ReactNode | null => { switch (statusToDisplay) { case "added": - return ; + return ; case "removed": - return ; + return ; case "changed": - return ; + return ; } return null; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/index.ts b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/index.ts deleted file mode 100644 index fc2c8b390ec..00000000000 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./StreamsConfigTableHeader"; -export * from "./StreamsConfigTableRow"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.test.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.test.tsx index 4fb42b6e5e0..4d2a9234a0d 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.test.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.test.tsx @@ -1,7 +1,6 @@ import { renderHook } from "@testing-library/react"; -import * as formik from "formik"; -import { FormikConnectionFormValues } from "components/connection/ConnectionForm/formConfig"; +import { HookFormConnectionFormValues } from "components/connection/ConnectionForm/hookFormConfig"; import { AirbyteStreamAndConfiguration } from "core/request/AirbyteClient"; import * as connectionFormService from "hooks/services/ConnectionForm/ConnectionFormService"; @@ -16,7 +15,7 @@ const mockStream: Partial = { config: { selected: true, syncMode: "full_refresh", destinationSyncMode: "overwrite" }, }; -const mockInitialValues: Partial = { +const mockInitialValues: Partial = { syncCatalog: { streams: [ { @@ -30,7 +29,7 @@ const mockInitialValues: Partial = { }, }; -const mockDisabledInitialValues: Partial = { +const mockDisabledInitialValues: Partial = { syncCatalog: { streams: [ { @@ -42,20 +41,16 @@ const mockDisabledInitialValues: Partial = { }, }; -const testSetup = (initialValues: Partial, error: unknown) => { +const testSetup = (initialValues: Partial) => { jest.spyOn(connectionFormService, "useConnectionFormService").mockImplementation(() => { // eslint-disable-next-line @typescript-eslint/no-explicit-any return { initialValues } as any; }); - jest.spyOn(formik, "useField").mockImplementationOnce(() => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return [{}, { error }] as any; // no error - }); }; describe(`${useStreamsConfigTableRowProps.name}`, () => { it("should return default styles for a row that starts enabled", () => { - testSetup(mockInitialValues, undefined); + testSetup(mockInitialValues); const { result } = renderHook(() => useStreamsConfigTableRowProps(mockStream)); @@ -63,7 +58,7 @@ describe(`${useStreamsConfigTableRowProps.name}`, () => { expect(result.current.pillButtonVariant).toEqual("grey"); }); it("should return disabled styles for a row that starts disabled", () => { - testSetup(mockDisabledInitialValues, undefined); + testSetup(mockDisabledInitialValues); const { result } = renderHook(() => useStreamsConfigTableRowProps({ @@ -77,7 +72,7 @@ describe(`${useStreamsConfigTableRowProps.name}`, () => { expect(result.current.pillButtonVariant).toEqual("grey"); }); it("should return added styles for a row that is added", () => { - testSetup(mockDisabledInitialValues, undefined); + testSetup(mockDisabledInitialValues); const { result } = renderHook(() => useStreamsConfigTableRowProps(mockStream)); @@ -85,7 +80,7 @@ describe(`${useStreamsConfigTableRowProps.name}`, () => { expect(result.current.pillButtonVariant).toEqual("green"); }); it("should return removed styles for a row that is removed", () => { - testSetup(mockInitialValues, undefined); + testSetup(mockInitialValues); const { result } = renderHook(() => useStreamsConfigTableRowProps({ @@ -99,8 +94,7 @@ describe(`${useStreamsConfigTableRowProps.name}`, () => { expect(result.current.pillButtonVariant).toEqual("red"); }); it("should return updated styles for a row that is updated", () => { - // eslint-disable-next-line - testSetup(mockInitialValues, undefined); + testSetup(mockInitialValues); const { result } = renderHook(() => useStreamsConfigTableRowProps({ @@ -115,7 +109,7 @@ describe(`${useStreamsConfigTableRowProps.name}`, () => { }); it("should return added styles for a row that is both added and updated", () => { - testSetup(mockDisabledInitialValues, undefined); + testSetup(mockDisabledInitialValues); const { result } = renderHook(() => useStreamsConfigTableRowProps({ diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.tsx index 6fa7de0520c..7f7b6d60e26 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.tsx @@ -4,8 +4,7 @@ import { useMemo } from "react"; import { PillButtonVariant } from "components/ui/PillListBox/PillButton"; -import { AirbyteStreamConfiguration } from "core/api/types/AirbyteClient"; -import { SyncSchemaStream } from "core/domain/catalog"; +import { AirbyteStreamAndConfiguration, AirbyteStreamConfiguration } from "core/api/types/AirbyteClient"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import styles from "./StreamsConfigTableRow.module.scss"; @@ -14,7 +13,7 @@ import { compareObjectsByFields } from "../utils"; export type StatusToDisplay = "disabled" | "added" | "removed" | "changed" | "unchanged"; -export const useStreamsConfigTableRowProps = (stream: SyncSchemaStream) => { +export const useStreamsConfigTableRowProps = (stream: AirbyteStreamAndConfiguration) => { const { initialValues } = useConnectionFormService(); const isStreamEnabled = stream.config?.selected; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalog.module.scss b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalog.module.scss deleted file mode 100644 index f2af20be9de..00000000000 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalog.module.scss +++ /dev/null @@ -1,12 +0,0 @@ -@use "scss/variables"; - -.bodyContainer { - flex: 1; - padding: 0; - margin-bottom: variables.$spacing-xl; - - &.scrollable { - overflow-y: auto; - max-height: 600px; - } -} diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalog.tsx b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalog.tsx deleted file mode 100644 index 0e2dd13a93e..00000000000 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalog.tsx +++ /dev/null @@ -1,69 +0,0 @@ -import classNames from "classnames"; -import React, { useCallback, useMemo, useState } from "react"; -import { useToggle } from "react-use"; - -import { LoadingBackdrop } from "components/ui/LoadingBackdrop"; - -import { SyncSchemaStream } from "core/domain/catalog"; -import { naturalComparatorBy } from "core/utils/objects"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; - -import { DisabledStreamsSwitch } from "./DisabledStreamsSwitch"; -import styles from "./SyncCatalog.module.scss"; -import { SyncCatalogBody } from "./SyncCatalogBody"; -import { SyncCatalogStreamSearch } from "./SyncCatalogStreamSearch"; -import { useStreamFilters } from "./useStreamFilters"; - -interface SyncCatalogProps { - streams: SyncSchemaStream[]; - onStreamsChanged: (streams: SyncSchemaStream[]) => void; - isLoading: boolean; -} -/** - * @deprecated will be removed during clean up - https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - * use SyncCatalogHookFormField.tsx instead - * @see SyncCatalogHookFormField - */ -const SyncCatalogInternal: React.FC> = ({ - streams, - onStreamsChanged, - isLoading, -}) => { - const { mode } = useConnectionFormService(); - - const [searchString, setSearchString] = useState(""); - const [hideDisabledStreams, toggleHideDisabledStreams] = useToggle(false); - - const onSingleStreamChanged = useCallback( - (newValue: SyncSchemaStream) => onStreamsChanged(streams.map((str) => (str.id === newValue.id ? newValue : str))), - [streams, onStreamsChanged] - ); - - const sortedSchema = useMemo( - () => [...streams].sort(naturalComparatorBy((syncStream) => syncStream.stream?.name ?? "")), - [streams] - ); - - const filteredStreams = useStreamFilters(searchString, hideDisabledStreams, sortedSchema); - - return ( - - - -
- -
- - ); -}; - -export const SyncCatalog = React.memo(SyncCatalogInternal); diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogBody.module.scss b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogBody.module.scss deleted file mode 100644 index 9d91759d73e..00000000000 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogBody.module.scss +++ /dev/null @@ -1,9 +0,0 @@ -@use "scss/colors"; -@use "scss/z-indices"; - -.header { - position: sticky; - top: 0; - z-index: z-indices.$mainPageWithScrollEdge; - background: colors.$foreground; -} diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogBody.tsx b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogBody.tsx deleted file mode 100644 index 72ca9611353..00000000000 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogBody.tsx +++ /dev/null @@ -1,113 +0,0 @@ -import { Field, FieldProps, setIn } from "formik"; -import React, { useCallback, useMemo } from "react"; -import { Location, useLocation } from "react-router-dom"; -import { IndexLocationWithAlign, Virtuoso } from "react-virtuoso"; - -import { FormikConnectionFormValues } from "components/connection/ConnectionForm/formConfig"; - -import { SyncSchemaStream } from "core/domain/catalog"; -import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; - -import styles from "./SyncCatalogBody.module.scss"; -import { SyncCatalogEmpty } from "./SyncCatalogEmpty"; -import { SyncCatalogRow } from "./SyncCatalogRow"; -import { StreamsConfigTableHeader } from "../StreamsConfigTable"; - -interface RedirectionLocationState { - namespace?: string; - streamName?: string; - action?: "showInReplicationTable" | "openDetails"; -} - -export interface LocationWithState extends Location { - state: RedirectionLocationState; -} - -interface SyncCatalogBodyProps { - streams: SyncSchemaStream[]; - onStreamsChanged: (streams: SyncSchemaStream[]) => void; - onStreamChanged: (stream: SyncSchemaStream) => void; - isFilterApplied?: boolean; -} -/** - * @deprecated will be removed during clean up - https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - * use SyncCatalogHookFormField.tsx instead - * @see SyncCatalogHookFormField - */ -export const SyncCatalogBody: React.FC = ({ - streams, - onStreamsChanged, - onStreamChanged, - isFilterApplied = false, -}) => { - const onUpdateStream = useCallback( - (id: string | undefined, newConfig: Partial) => { - const streamNode = streams.find((streamNode) => streamNode.id === id); - - if (streamNode) { - const newStreamNode = setIn(streamNode, "config", { ...streamNode.config, ...newConfig }); - - // config.selectedFields must be removed if fieldSelection is disabled - if (!newStreamNode.config.fieldSelectionEnabled) { - delete newStreamNode.config.selectedFields; - } - - onStreamChanged(newStreamNode); - } - }, - [streams, onStreamChanged] - ); - - // Scroll to the stream that was redirected from the Status tab - const { state: locationState } = useLocation() as LocationWithState; - const initialTopMostItemIndex: IndexLocationWithAlign | undefined = useMemo(() => { - if (locationState?.action !== "showInReplicationTable" && locationState?.action !== "openDetails") { - return; - } - - return { - index: streams.findIndex( - (stream) => - stream.stream?.name === locationState?.streamName && stream.stream?.namespace === locationState?.namespace - ), - align: "center", - }; - }, [locationState?.action, locationState?.namespace, locationState?.streamName, streams]); - - return ( -
-
- -
- {streams.length ? ( - ( - - {({ form }: FieldProps) => ( - - )} - - )} - /> - ) : ( - - )} -
- ); -}; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRow.tsx b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRow.tsx deleted file mode 100644 index eab93cf4dfc..00000000000 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRow.tsx +++ /dev/null @@ -1,230 +0,0 @@ -import { FormikErrors, getIn } from "formik"; -import React, { memo, useCallback, useMemo } from "react"; -import { useToggle } from "react-use"; - -import { ConnectionFormValues, SUPPORTED_MODES } from "components/connection/ConnectionForm/formConfig"; - -import { SyncSchemaField, SyncSchemaFieldObject, SyncSchemaStream } from "core/domain/catalog"; -import { traverseSchemaToField } from "core/domain/catalog/traverseSchemaToField"; -import { - AirbyteStreamConfiguration, - DestinationSyncMode, - NamespaceDefinitionType, - SyncMode, -} from "core/request/AirbyteClient"; -import { naturalComparatorBy } from "core/utils/objects"; -import { useDestinationNamespace } from "hooks/connection/useDestinationNamespace"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; - -import { - updatePrimaryKey, - toggleFieldInPrimaryKey, - updateCursorField, - updateFieldSelected, - toggleAllFieldsSelected, -} from "./streamConfigHelpers"; -import { updateStreamSyncMode } from "./updateStreamSyncMode"; -import { StreamDetailsPanel } from "../StreamDetailsPanel/StreamDetailsPanel"; -import { StreamsConfigTableRow } from "../StreamsConfigTable"; -import { SyncModeValue } from "../SyncModeSelect"; -import { flattenSyncSchemaFields, getFieldPathType } from "../utils"; - -interface SyncCatalogRowProps { - streamNode: SyncSchemaStream; - errors: FormikErrors; - namespaceDefinition: NamespaceDefinitionType; - namespaceFormat: string; - prefix: string; - updateStream: (id: string | undefined, newConfiguration: Partial) => void; -} - -const SyncCatalogRowInner: React.FC = ({ - streamNode, - updateStream, - namespaceDefinition, - namespaceFormat, - prefix, - errors, -}) => { - const { stream, config } = streamNode; - - const fields = useMemo(() => { - const traversedFields = traverseSchemaToField(stream?.jsonSchema, stream?.name); - return traversedFields.sort(naturalComparatorBy((field) => field.cleanedName)); - }, [stream?.jsonSchema, stream?.name]); - - // FIXME: Temp fix to return empty object when the json schema does not have .properties - // This prevents the table from crashing but still will not render the fields in the stream. - const streamProperties = streamNode?.stream?.jsonSchema?.properties ?? {}; - const numberOfFieldsInStream = Object.keys(streamProperties).length ?? 0; - - const { - destDefinitionSpecification: { supportedDestinationSyncModes }, - } = useConnectionFormService(); - const { mode } = useConnectionFormService(); - - const [isStreamDetailsPanelOpened, setIsStreamDetailsPanelOpened] = useToggle(false); - - const updateStreamWithConfig = useCallback( - (config: Partial) => updateStream(streamNode.id, config), - [updateStream, streamNode] - ); - - const onSelectSyncMode = useCallback( - (syncMode: SyncModeValue) => { - if (!streamNode.config || !streamNode.stream) { - return; - } - const updatedConfig = updateStreamSyncMode(streamNode.stream, streamNode.config, syncMode); - updateStreamWithConfig(updatedConfig); - }, - [streamNode, updateStreamWithConfig] - ); - - const onSelectStream = useCallback( - () => - updateStreamWithConfig({ - selected: !(config && config.selected), - }), - [config, updateStreamWithConfig] - ); - - const onPkSelect = useCallback( - (pkPath: string[]) => { - if (!config) { - return; - } - const updatedConfig = toggleFieldInPrimaryKey(config, pkPath, numberOfFieldsInStream); - updateStreamWithConfig(updatedConfig); - }, - [config, updateStreamWithConfig, numberOfFieldsInStream] - ); - - const onCursorSelect = useCallback( - (cursorField: string[]) => { - if (!config) { - return; - } - const updatedConfig = updateCursorField(config, cursorField, numberOfFieldsInStream); - updateStreamWithConfig(updatedConfig); - }, - [config, numberOfFieldsInStream, updateStreamWithConfig] - ); - - const onPkUpdate = useCallback( - (newPrimaryKey: string[][]) => { - if (!config) { - return; - } - const updatedConfig = updatePrimaryKey(config, newPrimaryKey, numberOfFieldsInStream); - updateStreamWithConfig(updatedConfig); - }, - [config, updateStreamWithConfig, numberOfFieldsInStream] - ); - - const onToggleAllFieldsSelected = useCallback(() => { - if (!config) { - return; - } - const updatedConfig = toggleAllFieldsSelected(config); - updateStreamWithConfig(updatedConfig); - }, [config, updateStreamWithConfig]); - - const onToggleFieldSelected = useCallback( - (fieldPath: string[], isSelected: boolean) => { - if (!config) { - return; - } - const updatedConfig = updateFieldSelected({ config, fields, fieldPath, isSelected, numberOfFieldsInStream }); - updateStreamWithConfig(updatedConfig); - }, - [config, fields, numberOfFieldsInStream, updateStreamWithConfig] - ); - - const pkRequired = config?.destinationSyncMode === DestinationSyncMode.append_dedup; - const cursorRequired = config?.syncMode === SyncMode.incremental; - const shouldDefinePk = stream?.sourceDefinedPrimaryKey?.length === 0 && pkRequired; - const shouldDefineCursor = !stream?.sourceDefinedCursor && cursorRequired; - - const availableSyncModes: SyncModeValue[] = useMemo( - () => - SUPPORTED_MODES.filter( - ([syncMode, destinationSyncMode]) => - stream?.supportedSyncModes?.includes(syncMode) && supportedDestinationSyncModes?.includes(destinationSyncMode) - ).map(([syncMode, destinationSyncMode]) => ({ - syncMode, - destinationSyncMode, - })), - [stream?.supportedSyncModes, supportedDestinationSyncModes] - ); - - const destNamespace = - useDestinationNamespace( - { - namespaceDefinition, - namespaceFormat, - }, - stream?.namespace - ) ?? ""; - - const flattenedFields = useMemo(() => flattenSyncSchemaFields(fields), [fields]); - - const primitiveFields = useMemo( - () => flattenedFields.filter(SyncSchemaFieldObject.isPrimitive), - [flattenedFields] - ); - - const destName = prefix + (streamNode.stream?.name ?? ""); - const configErrors = getIn(errors, `syncCatalog.streams[${streamNode.id}].config`); - const hasError = configErrors && Object.keys(configErrors).length > 0; - const pkType = getFieldPathType(pkRequired, shouldDefinePk); - const cursorType = getFieldPathType(cursorRequired, shouldDefineCursor); - const hasFields = fields?.length > 0; - const disabled = mode === "readonly"; - - return ( - <> - - - {isStreamDetailsPanelOpened && hasFields && ( - - )} - - ); -}; - -export const SyncCatalogRow = memo(SyncCatalogRowInner); diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRowHookForm.tsx b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRowHookForm.tsx index c5e8f2392b1..9d826bc7521 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRowHookForm.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRowHookForm.tsx @@ -1,11 +1,9 @@ import get from "lodash/get"; import set from "lodash/set"; import React, { useCallback, useMemo } from "react"; -import { FieldErrors } from "react-hook-form/dist/types/errors"; +import { FieldErrors } from "react-hook-form"; import { useToggle } from "react-use"; -import { SUPPORTED_MODES } from "components/connection/ConnectionForm/formConfig"; - import { SyncSchemaField, SyncSchemaFieldObject } from "core/domain/catalog"; import { traverseSchemaToField } from "core/domain/catalog/traverseSchemaToField"; import { AirbyteStreamConfiguration, DestinationSyncMode, SyncMode } from "core/request/AirbyteClient"; @@ -21,7 +19,11 @@ import { toggleAllFieldsSelected, } from "./streamConfigHelpers"; import { updateStreamSyncMode } from "./updateStreamSyncMode"; -import { HookFormConnectionFormValues, SyncStreamFieldWithId } from "../../ConnectionForm/hookFormConfig"; +import { + HookFormConnectionFormValues, + SyncStreamFieldWithId, + SUPPORTED_MODES, +} from "../../ConnectionForm/hookFormConfig"; import { StreamDetailsPanel } from "../StreamDetailsPanel/StreamDetailsPanel"; import { StreamsConfigTableRowHookForm } from "../StreamsConfigTable/StreamsConfigTableRowHookForm"; import { SyncModeValue } from "../SyncModeSelect"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/index.ts b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/index.ts deleted file mode 100644 index 6142c5066d2..00000000000 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./SyncCatalog"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/useStreamFilters.tsx b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/useStreamFilters.tsx index db01481b441..7eed4b2d365 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/useStreamFilters.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/useStreamFilters.tsx @@ -1,20 +1,21 @@ import { useMemo } from "react"; -import { SyncSchemaStream } from "core/domain/catalog"; +import { AirbyteStreamAndConfiguration } from "core/api/types/AirbyteClient"; export const useStreamFilters = ( searchString: string, hideDisabledStreams: boolean, - sortedSchema: SyncSchemaStream[] + sortedSchema: AirbyteStreamAndConfiguration[] ) => { return useMemo(() => { - const filters: Array<(s: SyncSchemaStream) => boolean> = [ - (_: SyncSchemaStream) => true, + const filters: Array<(s: AirbyteStreamAndConfiguration) => boolean> = [ + (_: AirbyteStreamAndConfiguration) => true, searchString - ? (stream: SyncSchemaStream) => stream.stream?.name.toLowerCase().includes(searchString.toLowerCase()) + ? (stream: AirbyteStreamAndConfiguration) => + stream.stream?.name.toLowerCase().includes(searchString.toLowerCase()) : null, - hideDisabledStreams ? (stream: SyncSchemaStream) => stream.config?.selected : null, - ].filter(Boolean) as Array<(s: SyncSchemaStream) => boolean>; + hideDisabledStreams ? (stream: AirbyteStreamAndConfiguration) => stream.config?.selected : null, + ].filter(Boolean) as Array<(s: AirbyteStreamAndConfiguration) => boolean>; return sortedSchema.filter((stream) => filters.every((f) => f(stream))); }, [hideDisabledStreams, searchString, sortedSchema]); diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss index 0900323d53d..91266cc70f0 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss @@ -21,7 +21,7 @@ $buttonWidth: 26px; display: flex; align-items: center; justify-content: center; - padding: 9px !important; + padding: 5px !important; z-index: 3; position: relative; } diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index 0927fbecb00..9c9612c9f36 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -1,23 +1,21 @@ import { yupResolver } from "@hookform/resolvers/yup"; import classNames from "classnames"; import merge from "lodash/merge"; -import { useMemo, useState } from "react"; -import React from "react"; +import React, { useMemo, useState } from "react"; import { FormProvider, useForm, useFormContext } from "react-hook-form"; import { FormattedMessage, useIntl } from "react-intl"; import { v4 as uuid } from "uuid"; import * as yup from "yup"; import { Button } from "components/ui/Button"; +import { Icon } from "components/ui/Icon"; import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import styles from "./AddStreamButton.module.scss"; import { BuilderField } from "./BuilderField"; import { BuilderFieldWithInputs } from "./BuilderFieldWithInputs"; -import PlusIcon from "../../connection/ConnectionOnboarding/plusIcon.svg?react"; import { BuilderStream, DEFAULT_BUILDER_STREAM_VALUES, @@ -113,7 +111,7 @@ export const AddStreamButton: React.FC = ({ type="button" className={styles.addButton} onClick={buttonClickHandler} - icon={} + icon={} data-testid={testId} />
@@ -131,6 +129,7 @@ export const AddStreamButton: React.FC = ({ onCancel={() => setIsOpen(false)} showCopyFromStream={!initialValues && numStreams > 0} streams={streams} + initialUrlPath={initialValues?.urlPath} /> )} @@ -143,15 +142,22 @@ const AddStreamForm = ({ onCancel, showCopyFromStream, streams, + initialUrlPath, }: { onSubmit: (values: AddStreamValues) => void; onCancel: () => void; showCopyFromStream: boolean; streams: BuilderStream[]; + initialUrlPath?: string; }) => { const { formatMessage } = useIntl(); const methods = useForm({ - defaultValues: { streamName: "", urlPath: "", copyOtherStream: false, streamToCopy: streams[0]?.name }, + defaultValues: { + streamName: "", + urlPath: initialUrlPath ?? "", + copyOtherStream: false, + streamToCopy: streams[0]?.name, + }, resolver: yupResolver( yup.object().shape({ streamName: yup diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx index 058263d7b12..49b58e4d825 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx @@ -1,3 +1,5 @@ +import { useIntl } from "react-intl"; + import GroupControls from "components/GroupControls"; import { ControlLabels } from "components/LabeledControl"; @@ -5,8 +7,7 @@ import { OAuthAuthenticatorRefreshTokenUpdater, SessionTokenAuthenticatorRequestAuthentication, } from "core/api/types/ConnectorManifest"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { links } from "core/utils/links"; import { BuilderCard } from "./BuilderCard"; @@ -15,8 +16,8 @@ import { BuilderFieldWithInputs } from "./BuilderFieldWithInputs"; import { BuilderInputPlaceholder } from "./BuilderInputPlaceholder"; import { BuilderOneOf } from "./BuilderOneOf"; import { BuilderOptional } from "./BuilderOptional"; +import { BuilderRequestInjection } from "./BuilderRequestInjection"; import { ErrorHandlerSection } from "./ErrorHandlerSection"; -import { InjectIntoFields } from "./InjectIntoFields"; import { KeyValueListField } from "./KeyValueListField"; import { getDescriptionByManifest, getLabelAndTooltip, getOptionsByManifest } from "./manifestHelpers"; import { RequestOptionSection } from "./RequestOptionSection"; @@ -44,13 +45,17 @@ const AUTH_PATH = "formValues.global.authenticator"; const authPath = (path: T) => `${AUTH_PATH}.${path}` as const; export const AuthenticationSection: React.FC = () => { + const { formatMessage } = useIntl(); const analyticsService = useAnalyticsService(); return ( - + path={AUTH_PATH} - label="Method" + label={formatMessage({ id: "connectorBuilder.authentication.method.label" })} manifestPath="HttpRequester.properties.authenticator" manifestOptionPaths={[ "ApiKeyAuthenticator", @@ -65,9 +70,9 @@ export const AuthenticationSection: React.FC = () => { }) } options={[ - { label: "No Auth", default: { type: NO_AUTH } }, + { label: formatMessage({ id: "connectorBuilder.authentication.method.noAuth" }), default: { type: NO_AUTH } }, { - label: "API Key", + label: formatMessage({ id: "connectorBuilder.authentication.method.apiKey" }), default: { type: API_KEY_AUTHENTICATOR, ...inferredAuthValues("ApiKeyAuthenticator"), @@ -79,13 +84,17 @@ export const AuthenticationSection: React.FC = () => { }, children: ( <> - + ), }, { - label: "Bearer", + label: formatMessage({ id: "connectorBuilder.authentication.method.bearer" }), default: { type: BEARER_AUTHENTICATOR, ...(inferredAuthValues("BearerAuthenticator") as Record<"api_token", string>), @@ -93,7 +102,7 @@ export const AuthenticationSection: React.FC = () => { children: , }, { - label: "Basic HTTP", + label: formatMessage({ id: "connectorBuilder.authentication.method.basicHttp" }), default: { type: BASIC_AUTHENTICATOR, ...(inferredAuthValues("BasicHttpAuthenticator") as Record<"username" | "password", string>), @@ -106,7 +115,7 @@ export const AuthenticationSection: React.FC = () => { ), }, { - label: "OAuth", + label: formatMessage({ id: "connectorBuilder.authentication.method.oAuth" }), default: { type: OAUTH_AUTHENTICATOR, ...(inferredAuthValues("OAuthAuthenticator") as Record< @@ -120,7 +129,7 @@ export const AuthenticationSection: React.FC = () => { children: , }, { - label: "Session Token", + label: formatMessage({ id: "connectorBuilder.authentication.method.sessionToken" }), default: { type: SESSION_TOKEN_AUTHENTICATOR, login_requester: { @@ -158,6 +167,7 @@ export const AuthenticationSection: React.FC = () => { }; const OAuthForm = () => { + const { formatMessage } = useIntl(); const grantType = useBuilderWatch(authPath("grant_type")); const refreshToken = useBuilderWatch(authPath("refresh_token")); return ( @@ -179,8 +189,8 @@ const OAuthForm = () => { <> - label="Overwrite config with refresh token response" - tooltip="If enabled, the refresh token response will overwrite the current OAuth config. This is useful if requesting a new access token invalidates the old refresh token." + label={formatMessage({ id: "connectorBuilder.authentication.refreshTokenUpdater.label" })} + tooltip={formatMessage({ id: "connectorBuilder.authentication.refreshTokenUpdater.tooltip" })} fieldPath={authPath("refresh_token_updater")} initialValues={{ refresh_token_name: "", @@ -233,8 +243,9 @@ const OAuthForm = () => { }; const SessionTokenForm = () => { + const { formatMessage } = useIntl(); const { label: loginRequesterLabel, tooltip: loginRequesterTooltip } = getLabelAndTooltip( - "Session Token Retrieval", + formatMessage({ id: "connectorBuilder.authentication.loginRequester.label" }), undefined, "SessionTokenAuthenticator.properties.login_requester", authPath("login_requester"), @@ -246,8 +257,8 @@ const SessionTokenForm = () => { { /> path={authPath("login_requester.authenticator")} - label="Authentication Method" + label={formatMessage({ id: "connectorBuilder.authentication.loginRequester.authenticator.label" })} manifestPath="HttpRequester.properties.authenticator" manifestOptionPaths={["ApiKeyAuthenticator", "BearerAuthenticator", "BasicHttpAuthenticator"]} options={[ - { label: "No Auth", default: { type: NO_AUTH } }, { - label: "API Key", + label: formatMessage({ id: "connectorBuilder.authentication.method.noAuth" }), + default: { type: NO_AUTH }, + }, + { + label: formatMessage({ id: "connectorBuilder.authentication.method.apiKey" }), default: { type: API_KEY_AUTHENTICATOR, ...inferredAuthValues("ApiKeyAuthenticator"), @@ -275,9 +289,9 @@ const SessionTokenForm = () => { }, children: ( <> - @@ -285,7 +299,7 @@ const SessionTokenForm = () => { ), }, { - label: "Bearer", + label: formatMessage({ id: "connectorBuilder.authentication.method.bearer" }), default: { type: BEARER_AUTHENTICATOR, ...(inferredAuthValues(BEARER_AUTHENTICATOR) as Record<"api_token", string>), @@ -293,7 +307,7 @@ const SessionTokenForm = () => { children: , }, { - label: "Basic HTTP", + label: formatMessage({ id: "connectorBuilder.authentication.method.basicHttp" }), default: { type: BASIC_AUTHENTICATOR, ...(inferredAuthValues(BASIC_AUTHENTICATOR) as Record<"username" | "password", string>), @@ -309,7 +323,7 @@ const SessionTokenForm = () => { /> - label="Error Handler" + label={formatMessage({ id: "connectorBuilder.authentication.loginRequester.errorHandler" })} tooltip={getDescriptionByManifest("DefaultErrorHandler")} fieldPath={authPath("login_requester.errorHandler")} initialValues={[{ type: "DefaultErrorHandler" }]} @@ -320,8 +334,8 @@ const SessionTokenForm = () => { { manifestOptionPaths={["SessionTokenRequestApiKeyAuthenticator", "SessionTokenRequestBearerAuthenticator"]} options={[ { - label: "API Key", + label: formatMessage({ id: "connectorBuilder.authentication.method.apiKey" }), default: { type: SESSION_TOKEN_REQUEST_API_KEY_AUTHENTICATOR, inject_into: { @@ -347,17 +361,19 @@ const SessionTokenForm = () => { }, }, children: ( - ), }, { - label: "Bearer", + label: formatMessage({ id: "connectorBuilder.authentication.method.bearer" }), default: { type: SESSION_TOKEN_REQUEST_BEARER_AUTHENTICATOR }, }, ]} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index 02b00d649de..f0970da4411 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -1,6 +1,5 @@ import debounce from "lodash/debounce"; -import { useEffect, useMemo } from "react"; -import React from "react"; +import React, { useEffect, useMemo } from "react"; import { removeEmptyProperties } from "core/utils/form"; import { useConnectorBuilderFormState } from "services/connectorBuilder/ConnectorBuilderStateService"; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx index 81253e93aa4..d4610ffaf42 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx @@ -1,5 +1,3 @@ -import { faArrowUpRightFromSquare } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import React, { useState } from "react"; import { FieldPath, useFormContext, useWatch } from "react-hook-form"; @@ -69,7 +67,7 @@ export const BuilderCard: React.FC> = rel="noreferrer" className={styles.docLink} > - + )} @@ -134,7 +132,7 @@ const CopyButtons = ({ copyConfig }: Pick) => { onClick={() => { setCopyFromOpen(true); }} - icon={} + icon={} /> {currentRelevantConfig && (
} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderInputPlaceholder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderInputPlaceholder.tsx index 0b55279b238..ea7334cf2e4 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderInputPlaceholder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderInputPlaceholder.tsx @@ -1,9 +1,8 @@ -import { faUser } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; import { InfoTooltip, Tooltip } from "components/ui/Tooltip"; @@ -29,7 +28,7 @@ export const BuilderInputPlaceholder = (props: BuilderFieldProps) => { {tooltip && {tooltip}} - }> + }>
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx index 59a0300b0d7..d71680621d0 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -1,3 +1,4 @@ +import lowerCase from "lodash/lowerCase"; import { useFormContext } from "react-hook-form"; import { useIntl } from "react-intl"; @@ -10,7 +11,7 @@ import { BuilderCard } from "./BuilderCard"; import { BuilderField } from "./BuilderField"; import { BuilderFieldWithInputs } from "./BuilderFieldWithInputs"; import { BuilderOneOf } from "./BuilderOneOf"; -import { InjectIntoFields } from "./InjectIntoFields"; +import { BuilderRequestInjection } from "./BuilderRequestInjection"; import { ToggleGroupField } from "./ToggleGroupField"; import { BuilderCursorPagination, @@ -36,8 +37,8 @@ export const PaginationSection: React.FC = ({ streamFiel return ( = ({ streamFiel > path={streamFieldPath("paginator.strategy")} - label="Mode" - tooltip="Pagination method to use for requests sent to the API" + label={formatMessage({ id: "connectorBuilder.pagination.strategy.label" })} + tooltip={formatMessage({ id: "connectorBuilder.pagination.strategy.tooltip" })} manifestOptionPaths={["OffsetIncrement", "PageIncrement", "CursorPagination"]} options={[ { - label: "Offset Increment", + label: formatMessage({ id: "connectorBuilder.pagination.strategy.offsetIncrement" }), default: { type: OFFSET_INCREMENT, page_size: "", @@ -83,13 +84,21 @@ export const PaginationSection: React.FC = ({ streamFiel path={streamFieldPath("paginator.strategy.page_size")} optional /> - {pageSize ? : null} - + {pageSize ? ( + + ) : null} + ), }, { - label: "Page Increment", + label: formatMessage({ id: "connectorBuilder.pagination.strategy.pageIncrement" }), default: { type: PAGE_INCREMENT, page_size: undefined, @@ -109,13 +118,21 @@ export const PaginationSection: React.FC = ({ streamFiel manifestPath="PageIncrement.properties.start_from_page" optional /> - {pageSize ? : null} - + {pageSize ? ( + + ) : null} + ), }, { - label: "Cursor Pagination", + label: formatMessage({ id: "connectorBuilder.pagination.strategy.cursor" }), default: { type: CURSOR_PAGINATION, page_size: undefined, @@ -128,32 +145,41 @@ export const PaginationSection: React.FC = ({ streamFiel <> path={streamFieldPath("paginator.strategy.cursor")} - label="Next page cursor" + label={formatMessage({ id: "connectorBuilder.pagination.strategy.cursor.cursor.label" })} tooltip={ } options={[ { - label: "Response", + label: formatMessage({ id: "connectorBuilder.pagination.strategy.cursor.cursor.response.label" }), default: { type: "response", path: [], @@ -162,8 +188,10 @@ export const PaginationSection: React.FC = ({ streamFiel ), }, @@ -177,13 +205,15 @@ export const PaginationSection: React.FC = ({ streamFiel ), }, { - label: "Custom", + label: formatMessage({ id: "connectorBuilder.pagination.strategy.cursor.cursor.custom.label" }), default: { type: "custom", cursor_value: "", @@ -207,7 +237,10 @@ export const PaginationSection: React.FC = ({ streamFiel }, ]} /> - + = ({ streamFiel }} optional /> - {pageSize ? : null} + {pageSize ? ( + + ) : null} ), }, @@ -236,10 +274,12 @@ const PageTokenOption = ({ label: string; streamFieldPath: (fieldPath: string) => string; }): JSX.Element => { + const { formatMessage } = useIntl(); + return ( - label={`Inject ${label} into outgoing HTTP Request`} - tooltip={`Configures how the ${label} will be sent in requests to the source API`} + label={formatMessage({ id: "connectorBuilder.injection.label" }, { label })} + tooltip={formatMessage({ id: "connectorBuilder.injection.tooltip" }, { label: lowerCase(label) })} fieldPath={streamFieldPath("paginator.pageTokenOption")} initialValues={{ inject_into: "request_parameter", @@ -247,7 +287,7 @@ const PageTokenOption = ({ field_name: "", }} > - + ); }; @@ -259,10 +299,12 @@ const PageSizeOption = ({ label: string; streamFieldPath: (fieldPath: string) => string; }): JSX.Element => { + const { formatMessage } = useIntl(); + return ( - label={`Inject ${label} into outgoing HTTP Request`} - tooltip={`Configures how the ${label} will be sent in requests to the source API`} + label={formatMessage({ id: "connectorBuilder.injection.label" }, { label })} + tooltip={formatMessage({ id: "connectorBuilder.injection.tooltip" }, { label: lowerCase(label) })} fieldPath={streamFieldPath("paginator.pageSizeOption")} initialValues={{ inject_into: "request_parameter", @@ -270,9 +312,9 @@ const PageSizeOption = ({ field_name: "", }} > - diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ParameterizedRequestsSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/ParameterizedRequestsSection.tsx index cbdaf75fca7..939ff0ce6e3 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/ParameterizedRequestsSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ParameterizedRequestsSection.tsx @@ -11,7 +11,7 @@ import { BuilderField } from "./BuilderField"; import { BuilderFieldWithInputs } from "./BuilderFieldWithInputs"; import { BuilderList } from "./BuilderList"; import { BuilderOneOf } from "./BuilderOneOf"; -import { InjectIntoFields } from "./InjectIntoFields"; +import { BuilderRequestInjection } from "./BuilderRequestInjection"; import { ToggleGroupField } from "./ToggleGroupField"; import { LIST_PARTITION_ROUTER, StreamPathFn, BuilderParameterizedRequests } from "../types"; @@ -35,8 +35,8 @@ export const ParameterizedRequestsSection: React.FC @@ -58,29 +58,39 @@ export const ParameterizedRequestsSection: React.FC path={buildPath("values")} manifestPath="ListPartitionRouter.properties.values" - label="Parameter Values" + label={formatMessage({ id: "connectorBuilder.parameterizedRequests.values" })} options={[ { - label: "Value List", + label: formatMessage({ id: "connectorBuilder.parameterizedRequests.values.list" }), default: { type: "list", value: [] }, - children: , + children: ( + + ), }, { - label: "User Input", + label: formatMessage({ id: "connectorBuilder.parameterizedRequests.values.userInput" }), default: { type: "variable", value: "" }, children: ( - { - "Reference an array user input here to allow the user to specify the values to iterate over, e.g. `{{ config['user_input_name'] }}`" - } + {formatMessage({ + id: "connectorBuilder.parameterizedRequests.values.userInput.value.tooltip", + })} } - pattern={"{{ config['user_input_name'] }}"} + pattern={formatMessage({ + id: "connectorBuilder.parameterizedRequests.values.userInput.value.pattern", + })} /> ), }, @@ -89,18 +99,16 @@ export const ParameterizedRequestsSection: React.FC - { - "The name of field used to reference a parameter value. The parameter value can be accessed with string interpolation, e.g. `{{ stream_partition['my_key'] }}` where `my_key` is the Current Parameter Value Identifier." - } + {formatMessage({ id: "connectorBuilder.parameterizedRequests.cursorField.tooltip" })} } /> - label="Inject Parameter Value into outgoing HTTP Request" - tooltip="Optionally configures how the parameter value will be sent in requests to the source API" + label={formatMessage({ id: "connectorBuilder.parameterizedRequests.requestOption.label" })} + tooltip={formatMessage({ id: "connectorBuilder.parameterizedRequests.requestOption.tooltip" })} fieldPath={buildPath("request_option")} initialValues={{ inject_into: "request_parameter", @@ -108,9 +116,9 @@ export const ParameterizedRequestsSection: React.FC - diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ParentStreamsSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/ParentStreamsSection.tsx index 6a226ff6018..68163b00a85 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/ParentStreamsSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ParentStreamsSection.tsx @@ -9,7 +9,7 @@ import { links } from "core/utils/links"; import { BuilderCard } from "./BuilderCard"; import { BuilderFieldWithInputs } from "./BuilderFieldWithInputs"; import { BuilderList } from "./BuilderList"; -import { InjectIntoFields } from "./InjectIntoFields"; +import { BuilderRequestInjection } from "./BuilderRequestInjection"; import { StreamReferenceField } from "./StreamReferenceField"; import { ToggleGroupField } from "./ToggleGroupField"; import { StreamPathFn, BuilderParentStream } from "../types"; @@ -31,8 +31,8 @@ export const ParentStreamsSection: React.FC = ({ stre return ( = ({ stre = ({ stre manifestPath="ParentStreamConfig.properties.partition_field" tooltip={ - { - "The identifier that should be used for referencing the parent key value in interpolation. For example, if this field is set to `parent_id`, then the parent key value can be referenced in interpolation as `{{ stream_partition.parent_id }}`" - } + {formatMessage({ id: "connectorBuilder.parentStreams.parentStream.partitionField.tooltip" })} } /> - label="Inject Parent Key into outgoing HTTP Request" - tooltip="Optionally configures how the parent key will be sent in requests to the source API" + label={formatMessage({ id: "connectorBuilder.parentStreams.parentStream.requestOption.label" })} + tooltip={formatMessage({ id: "connectorBuilder.parentStreams.parentStream.requestOption.tooltip" })} fieldPath={buildPath("request_option")} initialValues={{ inject_into: "request_parameter", @@ -84,7 +82,13 @@ export const ParentStreamsSection: React.FC = ({ stre field_name: "", }} > - + )} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/RequestOptionSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/RequestOptionSection.tsx index fbc320e1f48..0a442033526 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/RequestOptionSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/RequestOptionSection.tsx @@ -25,7 +25,7 @@ export const RequestOptionSection: React.FC = (props) const getBodyOptions = (): Array> => [ { - label: "JSON (key-value pairs)", + label: formatMessage({ id: "connectorBuilder.requestOptions.jsonList" }), default: { type: "json_list", values: [], @@ -41,7 +41,7 @@ export const RequestOptionSection: React.FC = (props) ), }, { - label: "Form encoded (key-value pairs)", + label: formatMessage({ id: "connectorBuilder.requestOptions.formList" }), default: { type: "form_list", values: [], @@ -57,7 +57,7 @@ export const RequestOptionSection: React.FC = (props) ), }, { - label: "JSON (free form)", + label: formatMessage({ id: "connectorBuilder.requestOptions.jsonFreeform" }), default: { type: "json_freeform", value: bodyValue.type === "json_list" ? JSON.stringify(Object.fromEntries(bodyValue.values)) : "{}", @@ -72,7 +72,7 @@ export const RequestOptionSection: React.FC = (props) ), }, { - label: "Text (Free form)", + label: formatMessage({ id: "connectorBuilder.requestOptions.stringFreeform" }), default: { type: "string_freeform", value: "", @@ -81,7 +81,7 @@ export const RequestOptionSection: React.FC = (props) @@ -105,7 +105,7 @@ export const RequestOptionSection: React.FC = (props) /> path={concatPath(props.basePath, "requestBody")} - label="Request Body" + label={formatMessage({ id: "connectorBuilder.requestOptions.requestBody" })} options={getBodyOptions()} omitInterpolationContext={props.omitInterpolationContext} /> diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/SavingIndicator.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/SavingIndicator.tsx index f63bd6cea92..b544311976a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/SavingIndicator.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/SavingIndicator.tsx @@ -1,10 +1,9 @@ -import { faCaretDown, faCheck } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { useEffect, useRef, useState } from "react"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; import { Tooltip } from "components/ui/Tooltip"; @@ -60,7 +59,7 @@ function getMessage(savingState: SavingState, displayedVersion: number | undefin } return ( - + {displayedVersion ? <>v{displayedVersion} : } @@ -114,7 +113,7 @@ export const SavingIndicator: React.FC = () => { onClick={() => { setChangeInProgress(true); }} - icon={} + icon={} iconPosition="right" > {message} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss index 858c12aee51..d3082b02ab2 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.module.scss @@ -75,6 +75,7 @@ $controlButtonWidth: 24px; .autoSchemaContainer { overflow-y: auto; flex: 1 1 0; + font-size: variables.$font-size-md; } .multiStreams { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index 2b95054c6f2..07b279c3725 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -1,19 +1,16 @@ -import { faTrashCan, faCopy } from "@fortawesome/free-regular-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; -import { useCallback, useMemo, useState } from "react"; -import React from "react"; +import React, { useCallback, useMemo, useState } from "react"; import { get, useFormContext, useFormState } from "react-hook-form"; import { FormattedMessage, useIntl } from "react-intl"; import Indicator from "components/Indicator"; import { Button } from "components/ui/Button"; import { CodeEditor } from "components/ui/CodeEditor"; +import { Icon } from "components/ui/Icon"; import { Pre } from "components/ui/Pre"; import { Text } from "components/ui/Text"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { BuilderView, useConnectorBuilderTestRead } from "services/connectorBuilder/ConnectorBuilderStateService"; @@ -58,7 +55,11 @@ export const StreamConfigView: React.FC = React.memo(({ s className={hasMultipleStreams ? styles.multiStreams : undefined} > {/* Not using intl for the labels and tooltips in this component in order to keep maintainence simple */} - + = React.memo(({ s @@ -190,13 +191,13 @@ const StreamControls = ({ initialValues={streams[streamNum]} button={ } modalTitle={formatMessage({ id: "connectorBuilder.copyStreamModal.title" }, { name: streams[streamNum].name })} />
); @@ -260,7 +261,7 @@ const SchemaEditor = ({ streamFieldPath }: { streamFieldPath: StreamPathFn }) => return ( <>
}} />} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/TransformationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/TransformationSection.tsx index 1aaa9325b81..2c3d8dacfae 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/TransformationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/TransformationSection.tsx @@ -24,17 +24,22 @@ export const TransformationSection: React.FC = ({ const getTransformationOptions = (buildPath: (path: string) => string): Array> => [ { - label: "Remove field", + label: formatMessage({ id: "connectorBuilder.transformation.remove" }), default: { type: "remove", path: [], }, children: ( - + ), }, { - label: "Add field", + label: formatMessage({ id: "connectorBuilder.transformation.add" }), default: { type: "add", value: "", @@ -85,8 +90,8 @@ export const TransformationSection: React.FC = ({ {({ buildPath }) => ( path={buildPath("")} - label="Transformation" - tooltip="Add or remove a field" + label={formatMessage({ id: "connectorBuilder.transformation.label" })} + tooltip={formatMessage({ id: "connectorBuilder.transformation.tooltip" })} options={getTransformationOptions(buildPath)} /> )} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx index f8846ed47b3..ae7c70ba5c0 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx @@ -3,8 +3,7 @@ import { FormattedMessage } from "react-intl"; import { Text } from "components/ui/Text"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import styles from "./UiYamlToggleButton.module.scss"; diff --git a/airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx b/airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx index 348bbfb971b..a5b2b6eb9cc 100644 --- a/airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx @@ -1,14 +1,12 @@ -import { faWarning } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { dump } from "js-yaml"; import snakeCase from "lodash/snakeCase"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; +import { Icon } from "components/ui/Icon"; import { Tooltip } from "components/ui/Tooltip"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { FILE_TYPE_DOWNLOAD, downloadFile } from "core/utils/file"; import { useConnectorBuilderFormState } from "services/connectorBuilder/ConnectorBuilderStateService"; @@ -73,7 +71,7 @@ export const DownloadYamlButton: React.FC = ({ classNam variant="secondary" onClick={handleClick} disabled={buttonDisabled} - icon={showWarningIcon ? : undefined} + icon={showWarningIcon ? : undefined} data-testid="download-yaml-button" type="button" > diff --git a/airbyte-webapp/src/components/connectorBuilder/PublishButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/PublishButton.module.scss new file mode 100644 index 00000000000..b2f670d343a --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/PublishButton.module.scss @@ -0,0 +1,3 @@ +.tooltipContainer { + width: 100%; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/PublishButton.tsx b/airbyte-webapp/src/components/connectorBuilder/PublishButton.tsx index 8a22ee26660..0cee65a9b4f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/PublishButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/PublishButton.tsx @@ -1,9 +1,8 @@ -import { faWarning } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { useState } from "react"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; +import { Icon } from "components/ui/Icon"; import { Tooltip } from "components/ui/Tooltip"; import { @@ -11,6 +10,7 @@ import { useConnectorBuilderTestRead, } from "services/connectorBuilder/ConnectorBuilderStateService"; +import styles from "./PublishButton.module.scss"; import { PublishModal } from "./PublishModal"; import { useBuilderWatch } from "./types"; @@ -57,7 +57,7 @@ export const PublishButton: React.FC = ({ className }) => { }} disabled={buttonDisabled} data-testid="publish-button" - icon={showWarningIcon ? : undefined} + icon={showWarningIcon ? : undefined} type="button" > = ({ className }) => { return (
{tooltipContent !== undefined ? ( - + {tooltipContent} ) : ( diff --git a/airbyte-webapp/src/components/connectorBuilder/PublishModal.tsx b/airbyte-webapp/src/components/connectorBuilder/PublishModal.tsx index bcc7277300e..767235cef11 100644 --- a/airbyte-webapp/src/components/connectorBuilder/PublishModal.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/PublishModal.tsx @@ -11,8 +11,7 @@ import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; import { Spinner } from "components/ui/Spinner"; import { useListBuilderProjectVersions } from "core/api"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useNotificationService } from "hooks/services/Notification"; import { useConnectorBuilderFormState } from "services/connectorBuilder/ConnectorBuilderStateService"; diff --git a/airbyte-webapp/src/components/connectorBuilder/SchemaConflictIndicator.tsx b/airbyte-webapp/src/components/connectorBuilder/SchemaConflictIndicator.tsx index d9232722b4f..54fa125e769 100644 --- a/airbyte-webapp/src/components/connectorBuilder/SchemaConflictIndicator.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/SchemaConflictIndicator.tsx @@ -1,7 +1,6 @@ -import { faWarning } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; +import { Icon } from "components/ui/Icon"; import { Tooltip } from "components/ui/Tooltip"; import styles from "./SchemaConflictIndicator.module.scss"; @@ -10,8 +9,8 @@ import { SchemaConflictMessage } from "./SchemaConflictMessage"; export const SchemaConflictIndicator: React.FC<{ errors?: string[] }> = ({ errors }) => ( = ({ testInputJsonErrors, isO !jsonManifest.spec || Object.keys(jsonManifest.spec.connection_specification?.properties || {}).length === 0 } - icon={} + icon={} > diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/InnerListBox.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/InnerListBox.tsx index ba08f0375da..91d04e29502 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/InnerListBox.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/InnerListBox.tsx @@ -1,6 +1,4 @@ -import { faAngleDown } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; - +import { Icon } from "components/ui/Icon"; import { ListBox, ListBoxControlButtonProps, ListBoxProps } from "components/ui/ListBox"; import { Text } from "components/ui/Text"; @@ -10,7 +8,7 @@ const ControlButton = (props: ListBoxControlButtonProps) => { return ( <> {props.selectedOption && {props.selectedOption.label}} - + ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx index d4886c77a6a..aac54ec8232 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx @@ -1,10 +1,9 @@ -import { faTable } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { useMemo } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { Pre } from "components/ui/Pre"; import { Text } from "components/ui/Text"; import { InfoTooltip, Tooltip } from "components/ui/Tooltip"; @@ -129,7 +128,7 @@ const RecordDisplay = ({ records }: { records: StreamReadSlicesItemPagesItemReco setRecordViewMode("table"); }} > - + } > diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/RecordTable.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/RecordTable.tsx index af6679f7d47..a86cc303ef6 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/RecordTable.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/RecordTable.tsx @@ -1,10 +1,9 @@ -import { faExpand } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { createColumnHelper } from "@tanstack/react-table"; import { useMemo, useState } from "react"; import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { Modal, ModalBody } from "components/ui/Modal"; import { Pre } from "components/ui/Pre"; import { Table } from "components/ui/Table"; @@ -57,7 +56,7 @@ const ExpandableDataCell = ({ value, selectValue }: { value: unknown; selectValu
{stringRepresentation}
 ); diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SchemaDiffView.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SchemaDiffView.tsx index 04235558007..d44cdcd0778 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SchemaDiffView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SchemaDiffView.tsx @@ -13,8 +13,7 @@ import { Pre } from "components/ui/Pre"; import { Tooltip } from "components/ui/Tooltip"; import { StreamReadInferredSchema } from "core/api/types/ConnectorBuilderClient"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useConnectorBuilderTestRead } from "services/connectorBuilder/ConnectorBuilderStateService"; import styles from "./SchemaDiffView.module.scss"; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SliceSelector.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SliceSelector.tsx index e5433754060..0a6b81fdaa2 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SliceSelector.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SliceSelector.tsx @@ -1,5 +1,4 @@ -import { useMemo } from "react"; -import React from "react"; +import React, { useMemo } from "react"; import { useIntl } from "react-intl"; import { FlexContainer, FlexItem } from "components/ui/Flex"; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx index 51ae9d650df..b1465bb1128 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx @@ -8,8 +8,7 @@ import { Heading } from "components/ui/Heading"; import { Icon } from "components/ui/Icon"; import { ListBox, ListBoxControlButtonProps } from "components/ui/ListBox"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useConnectorBuilderTestRead } from "services/connectorBuilder/ConnectorBuilderStateService"; import styles from "./StreamSelector.module.scss"; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx index 9d0db69383b..405235380b6 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx @@ -1,9 +1,7 @@ -import { faWarning } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { FormattedMessage } from "react-intl"; -import { RotateIcon } from "components/icons/RotateIcon"; import { Button } from "components/ui/Button"; +import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; import { Tooltip } from "components/ui/Tooltip"; @@ -80,10 +78,10 @@ export const StreamTestButton: React.FC = ({ data-testid="read-stream" icon={ showWarningIcon ? ( - + ) : (
- +
) } diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss index c2b61f22912..a216c2831c8 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss @@ -8,6 +8,7 @@ gap: variables.$spacing-lg; min-height: 0; width: 100%; + font-size: variables.$font-size-md; } .resizablePanelsContainer { diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx index 510e90b597d..64681c6da9f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx @@ -14,8 +14,7 @@ import { Text } from "components/ui/Text"; import { KnownExceptionInfo } from "core/api/types/ConnectorBuilderClient"; import { CommonRequestError } from "core/request/CommonRequestError"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { links } from "core/utils/links"; import { useLocalStorage } from "core/utils/useLocalStorage"; import { useConnectorBuilderTestRead } from "services/connectorBuilder/ConnectorBuilderStateService"; @@ -84,8 +83,9 @@ export const StreamTester: React.FC<{ const hasRegularRequests = streamReadData !== undefined && !isError && streamReadData.slices && streamReadData.slices.length > 0; - const logsFlex = isError || errorLogs.length > 0 ? 0.75 : 0; - const auxiliaryRequestsFlex = hasAuxiliaryRequests && !hasRegularRequests ? 0.75 : 0; + const SECONDARY_PANEL_SIZE = 0.5; + const logsFlex = isError || errorLogs.length > 0 ? SECONDARY_PANEL_SIZE : 0; + const auxiliaryRequestsFlex = hasAuxiliaryRequests && !hasRegularRequests ? SECONDARY_PANEL_SIZE : 0; useEffect(() => { // This will only be true if the data was manually refetched by the user clicking the Test button, @@ -202,7 +202,13 @@ export const StreamTester: React.FC<{ children: , minWidth: 0, flex: logsFlex, - splitter: , + splitter: ( + + ), className: styles.secondaryPanel, }, ] diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx index 84868973a02..581db3de762 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx @@ -2,8 +2,7 @@ import { useMonaco } from "@monaco-editor/react"; import { load, YAMLException } from "js-yaml"; import debounce from "lodash/debounce"; import { editor } from "monaco-editor/esm/vs/editor/editor.api"; -import { useMemo, useRef } from "react"; -import React from "react"; +import React, { useMemo, useRef } from "react"; import { useFormContext } from "react-hook-form"; import { useUpdateEffect } from "react-use"; diff --git a/airbyte-webapp/src/components/forms/FormControl.tsx b/airbyte-webapp/src/components/forms/FormControl.tsx index e33dc578640..328302cf5b9 100644 --- a/airbyte-webapp/src/components/forms/FormControl.tsx +++ b/airbyte-webapp/src/components/forms/FormControl.tsx @@ -1,7 +1,6 @@ import classNames from "classnames"; import uniqueId from "lodash/uniqueId"; -import { HTMLInputTypeAttribute, ReactNode, useState } from "react"; -import React from "react"; +import React, { HTMLInputTypeAttribute, ReactNode, useState } from "react"; import { FieldError, Path, get, useFormState } from "react-hook-form"; import { useIntl } from "react-intl"; diff --git a/airbyte-webapp/src/components/icons/ArrowRightIcon.tsx b/airbyte-webapp/src/components/icons/ArrowRightIcon.tsx deleted file mode 100644 index d1d897f2773..00000000000 --- a/airbyte-webapp/src/components/icons/ArrowRightIcon.tsx +++ /dev/null @@ -1,8 +0,0 @@ -export const ArrowRightIcon = ({ color = "currentColor" }: { color?: string }) => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/ClockIcon.tsx b/airbyte-webapp/src/components/icons/ClockIcon.tsx deleted file mode 100644 index 48bcebb1c13..00000000000 --- a/airbyte-webapp/src/components/icons/ClockIcon.tsx +++ /dev/null @@ -1,8 +0,0 @@ -export const ClockIcon: React.FC<{ className?: string; viewBox?: string }> = ({ className, viewBox = "0 0 24 24" }) => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/CreditsIcon.tsx b/airbyte-webapp/src/components/icons/CreditsIcon.tsx deleted file mode 100644 index 863cc661fe4..00000000000 --- a/airbyte-webapp/src/components/icons/CreditsIcon.tsx +++ /dev/null @@ -1,14 +0,0 @@ -export const CreditsIcon = () => ( - - - - -); diff --git a/airbyte-webapp/src/components/icons/CrossIcon.tsx b/airbyte-webapp/src/components/icons/CrossIcon.tsx deleted file mode 100644 index 3153d654923..00000000000 --- a/airbyte-webapp/src/components/icons/CrossIcon.tsx +++ /dev/null @@ -1,14 +0,0 @@ -interface CrossIconProps { - color?: string; - title?: string; -} - -export const CrossIcon = ({ color = "currentColor", title }: CrossIconProps) => ( - - {title && {title}} - - -); diff --git a/airbyte-webapp/src/components/icons/DocsIcon.tsx b/airbyte-webapp/src/components/icons/DocsIcon.tsx deleted file mode 100644 index 9fcd7481a76..00000000000 --- a/airbyte-webapp/src/components/icons/DocsIcon.tsx +++ /dev/null @@ -1,8 +0,0 @@ -export const DocsIcon = ({ color = "currentColor" }: { color?: string }): JSX.Element => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/ErrorIcon.tsx b/airbyte-webapp/src/components/icons/ErrorIcon.tsx deleted file mode 100644 index f229b15bff6..00000000000 --- a/airbyte-webapp/src/components/icons/ErrorIcon.tsx +++ /dev/null @@ -1,10 +0,0 @@ -export const ErrorIcon: React.FC<{ className?: string }> = ({ className }) => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/InfoIcon.tsx b/airbyte-webapp/src/components/icons/InfoIcon.tsx deleted file mode 100644 index 24cd7f08cfd..00000000000 --- a/airbyte-webapp/src/components/icons/InfoIcon.tsx +++ /dev/null @@ -1,8 +0,0 @@ -export const InfoIcon = ({ color = "currentColor" }: { color?: string }): JSX.Element => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/MinusIcon.tsx b/airbyte-webapp/src/components/icons/MinusIcon.tsx deleted file mode 100644 index 24755afd617..00000000000 --- a/airbyte-webapp/src/components/icons/MinusIcon.tsx +++ /dev/null @@ -1,9 +0,0 @@ -interface MinusIconProps { - color?: string; -} - -export const MinusIcon: React.FC = ({ color = "currentColor" }) => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/ModificationIcon.tsx b/airbyte-webapp/src/components/icons/ModificationIcon.tsx deleted file mode 100644 index c0867a94369..00000000000 --- a/airbyte-webapp/src/components/icons/ModificationIcon.tsx +++ /dev/null @@ -1,20 +0,0 @@ -interface ModificationIconProps { - className?: string; -} - -export const ModificationIcon: React.FC = ({ className }) => { - return ( - - - - - ); -}; diff --git a/airbyte-webapp/src/components/icons/MoonIcon.tsx b/airbyte-webapp/src/components/icons/MoonIcon.tsx deleted file mode 100644 index 06b36d1aac1..00000000000 --- a/airbyte-webapp/src/components/icons/MoonIcon.tsx +++ /dev/null @@ -1,15 +0,0 @@ -interface MoonProps { - title?: string; -} - -export const MoonIcon = ({ title }: MoonProps): JSX.Element => ( - - {title && {title}} - - -); diff --git a/airbyte-webapp/src/components/icons/PauseIcon.tsx b/airbyte-webapp/src/components/icons/PauseIcon.tsx deleted file mode 100644 index a608fbf621b..00000000000 --- a/airbyte-webapp/src/components/icons/PauseIcon.tsx +++ /dev/null @@ -1,12 +0,0 @@ -interface Props { - color?: string; - title?: string; -} - -export const PauseIcon = ({ color = "currentColor", title }: Props): JSX.Element => ( - - {title && {title}} - - - -); diff --git a/airbyte-webapp/src/components/icons/PlusIcon.tsx b/airbyte-webapp/src/components/icons/PlusIcon.tsx deleted file mode 100644 index 027366b8f00..00000000000 --- a/airbyte-webapp/src/components/icons/PlusIcon.tsx +++ /dev/null @@ -1,12 +0,0 @@ -interface PlusIconProps { - color?: string; -} - -export const PlusIcon: React.FC = ({ color = "currentColor" }) => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/RotateIcon.tsx b/airbyte-webapp/src/components/icons/RotateIcon.tsx deleted file mode 100644 index b6a2e81372d..00000000000 --- a/airbyte-webapp/src/components/icons/RotateIcon.tsx +++ /dev/null @@ -1,15 +0,0 @@ -interface Props { - color?: string; - width?: string; - height?: string; - className?: string; -} - -export const RotateIcon = ({ width = "20", height = "20", className }: Props) => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/SimpleCircleIcon.tsx b/airbyte-webapp/src/components/icons/SimpleCircleIcon.tsx deleted file mode 100644 index 33607a00a2a..00000000000 --- a/airbyte-webapp/src/components/icons/SimpleCircleIcon.tsx +++ /dev/null @@ -1,13 +0,0 @@ -interface SimpleCircleIconProps { - className?: string; - viewBox?: string; -} - -export const SimpleCircleIcon: React.FC = ({ className, viewBox = "0 0 24 24" }) => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/SuccessIcon.tsx b/airbyte-webapp/src/components/icons/SuccessIcon.tsx deleted file mode 100644 index 33fbb4959a8..00000000000 --- a/airbyte-webapp/src/components/icons/SuccessIcon.tsx +++ /dev/null @@ -1,12 +0,0 @@ -interface SuccessIconProps { - className?: string; -} - -export const SuccessIcon: React.FC = ({ className }) => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/SunIcon.tsx b/airbyte-webapp/src/components/icons/SunIcon.tsx deleted file mode 100644 index 5716e78f7b3..00000000000 --- a/airbyte-webapp/src/components/icons/SunIcon.tsx +++ /dev/null @@ -1,23 +0,0 @@ -export const SunIcon = ({ className }: { className?: string }) => ( - - - - - - - - - - - -); diff --git a/airbyte-webapp/src/components/icons/TimeIcon.tsx b/airbyte-webapp/src/components/icons/TimeIcon.tsx deleted file mode 100644 index 24ca0bcb3e1..00000000000 --- a/airbyte-webapp/src/components/icons/TimeIcon.tsx +++ /dev/null @@ -1,16 +0,0 @@ -export const TimeIcon: React.FC = () => ( - - - - - - - - - - - -); diff --git a/airbyte-webapp/src/components/icons/WarningCircleIcon.tsx b/airbyte-webapp/src/components/icons/WarningCircleIcon.tsx deleted file mode 100644 index d0c51404703..00000000000 --- a/airbyte-webapp/src/components/icons/WarningCircleIcon.tsx +++ /dev/null @@ -1,10 +0,0 @@ -export const WarningCircleIcon: React.FC<{ className?: string }> = ({ className }) => ( - - - -); diff --git a/airbyte-webapp/src/components/icons/icon.module.scss b/airbyte-webapp/src/components/icons/icon.module.scss deleted file mode 100644 index ee99f7ae4c0..00000000000 --- a/airbyte-webapp/src/components/icons/icon.module.scss +++ /dev/null @@ -1,13 +0,0 @@ -@use "scss/colors"; - -.icon { - vertical-align: middle; -} - -.grey300 { - color: colors.$grey-300; -} - -.white { - color: colors.$foreground; -} diff --git a/airbyte-webapp/src/components/source/SelectConnector/RequestNewConnectorButton.tsx b/airbyte-webapp/src/components/source/SelectConnector/RequestNewConnectorButton.tsx index 5c91efd3119..c3ba7f79bb8 100644 --- a/airbyte-webapp/src/components/source/SelectConnector/RequestNewConnectorButton.tsx +++ b/airbyte-webapp/src/components/source/SelectConnector/RequestNewConnectorButton.tsx @@ -1,6 +1,6 @@ import { FormattedMessage } from "react-intl"; -import { PlusIcon } from "components/icons/PlusIcon"; +import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; import styles from "./RequestNewConnectorButton.module.scss"; @@ -12,7 +12,7 @@ interface RequestNewConnectorButtonProps { export const RequestNewConnectorButton: React.FC = ({ onClick }) => { return (
{label} {showErrorIndicator && } diff --git a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx b/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx index 99a235fc174..c86dca23f33 100644 --- a/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx +++ b/airbyte-webapp/src/components/ui/CollapsibleCard/CollapsibleCard.tsx @@ -35,7 +35,7 @@ export const CollapsibleCard: React.FC = ({ } const headerContainer = ( - + {title && ( {title} @@ -47,7 +47,6 @@ export const CollapsibleCard: React.FC = ({ size="lg" color="affordance" className={classNames(styles.icon, { [styles.expanded]: !isCollapsed })} - data-testid={`${testId}-card-expand-arrow`} /> )} diff --git a/airbyte-webapp/src/components/ui/ConnectorDefinitionBranding/ConnectorDefinitionBranding.tsx b/airbyte-webapp/src/components/ui/ConnectorDefinitionBranding/ConnectorDefinitionBranding.tsx index 89b94eb4f13..bb8dbc21130 100644 --- a/airbyte-webapp/src/components/ui/ConnectorDefinitionBranding/ConnectorDefinitionBranding.tsx +++ b/airbyte-webapp/src/components/ui/ConnectorDefinitionBranding/ConnectorDefinitionBranding.tsx @@ -1,9 +1,8 @@ import { ConnectorIcon } from "components/common/ConnectorIcon"; import { Text } from "components/ui/Text"; +import { useSourceDefinitionList, useDestinationDefinitionList } from "core/api"; import { DestinationDefinitionId, SourceDefinitionId } from "core/request/AirbyteClient"; -import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; -import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; import styles from "./ConnectorDefinitionBranding.module.scss"; import { FlexContainer } from "../Flex"; diff --git a/airbyte-webapp/src/components/ui/CopyButton/CopyButton.tsx b/airbyte-webapp/src/components/ui/CopyButton/CopyButton.tsx index 866e87a7b5f..b1b93c8b38f 100644 --- a/airbyte-webapp/src/components/ui/CopyButton/CopyButton.tsx +++ b/airbyte-webapp/src/components/ui/CopyButton/CopyButton.tsx @@ -1,5 +1,3 @@ -import { faCopy } from "@fortawesome/free-regular-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import { useRef, useState } from "react"; import { useIntl } from "react-intl"; @@ -39,8 +37,8 @@ export const CopyButton: React.FC = ({ className, content, titl title={title || formatMessage({ id: "copyButton.title" })} icon={
- - {copied && } + + {copied && }
} onClick={handleClick} diff --git a/airbyte-webapp/src/components/ui/DatePicker/DatePicker.tsx b/airbyte-webapp/src/components/ui/DatePicker/DatePicker.tsx index b296062982c..0453b394d5e 100644 --- a/airbyte-webapp/src/components/ui/DatePicker/DatePicker.tsx +++ b/airbyte-webapp/src/components/ui/DatePicker/DatePicker.tsx @@ -1,5 +1,3 @@ -import { faCalendarAlt } from "@fortawesome/free-regular-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import en from "date-fns/locale/en-US"; import dayjs from "dayjs"; import React, { useCallback, useEffect, useMemo, useRef } from "react"; @@ -7,6 +5,8 @@ import ReactDatePicker, { registerLocale } from "react-datepicker"; import "react-datepicker/dist/react-datepicker.css"; import { useIntl } from "react-intl"; +import { Icon } from "components/ui/Icon"; + import styles from "./DatePicker.module.scss"; import { Button } from "../Button"; import { Input } from "../Input"; @@ -78,7 +78,7 @@ const DatepickerButton = React.forwardRef} + icon={} /> ); }); diff --git a/airbyte-webapp/src/components/ui/DropDown/components/DropdownIndicator.tsx b/airbyte-webapp/src/components/ui/DropDown/components/DropdownIndicator.tsx index fb6c742b5ff..ba3ed9e3e85 100644 --- a/airbyte-webapp/src/components/ui/DropDown/components/DropdownIndicator.tsx +++ b/airbyte-webapp/src/components/ui/DropDown/components/DropdownIndicator.tsx @@ -1,15 +1,10 @@ -import { faSortDown } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import { components, DropdownIndicatorProps } from "react-select"; -import styled from "styled-components"; -const Arrow = styled(FontAwesomeIcon)` - margin-top: -6px; -`; +import { Icon } from "components/ui/Icon"; export const DropdownIndicator: React.FC = (props) => ( - + ); diff --git a/airbyte-webapp/src/components/ui/FileUpload/FileUpload.tsx b/airbyte-webapp/src/components/ui/FileUpload/FileUpload.tsx index 8fa26c0f5b3..096f05f7799 100644 --- a/airbyte-webapp/src/components/ui/FileUpload/FileUpload.tsx +++ b/airbyte-webapp/src/components/ui/FileUpload/FileUpload.tsx @@ -1,8 +1,8 @@ -import { faFileLines } from "@fortawesome/free-regular-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import { useRef, useState } from "react"; +import { Icon } from "components/ui/Icon"; + import styles from "./FileUpload.module.scss"; import { FlexContainer } from "../Flex"; import { Text } from "../Text"; @@ -79,7 +79,7 @@ export const FileUpload: React.FC = ({ onUpload }) => { className={classNames(styles.button, { [styles.draggingOver]: isDraggingOver })} > - + e.preventDefault()}> Upload file diff --git a/airbyte-webapp/src/components/ui/Icon/Icon.module.scss b/airbyte-webapp/src/components/ui/Icon/Icon.module.scss index d7a325b51ba..f90d75b73fa 100644 --- a/airbyte-webapp/src/components/ui/Icon/Icon.module.scss +++ b/airbyte-webapp/src/components/ui/Icon/Icon.module.scss @@ -39,8 +39,12 @@ $iconClassname: "icon"; } } -$sizes: "xs" calc(1 / 2.618) 12% -6%, "sm" calc(1 / 1.618) 3% -13%, "md" 1 -6% -30%, "lg" 1.618 -27% -41%, - "xl" 2.618 -60% -73%; +$sizes: + "xs" calc(1 / 1.618) 3% -13%, + "sm" 1 -6% -30%, + "md" 1.618 -27% -41%, + "lg" 2.618 -60% -73%, + "xl" 3.618 -90% -93%; @each $name, $sizeModifier, $verticalAlign, $verticalAlignWithBackground in $sizes { .#{$name} { diff --git a/airbyte-webapp/src/components/ui/Icon/Icon.tsx b/airbyte-webapp/src/components/ui/Icon/Icon.tsx index 0cb1a984ced..f429bef2f10 100644 --- a/airbyte-webapp/src/components/ui/Icon/Icon.tsx +++ b/airbyte-webapp/src/components/ui/Icon/Icon.tsx @@ -1,80 +1,130 @@ import classNames from "classnames"; +import kebabCase from "lodash/kebabCase"; import React from "react"; import styles from "./Icon.module.scss"; +import AddCircleIcon from "./icons/addCircleIcon.svg?react"; import ArrowLeftIcon from "./icons/arrowLeftIcon.svg?react"; import ArrowRightIcon from "./icons/arrowRightIcon.svg?react"; import ArticleIcon from "./icons/articleIcon.svg?react"; +import BellIcon from "./icons/bellIcon.svg?react"; +import CalendarCheckIcon from "./icons/calendarCheckIcon.svg?react"; import CalendarIcon from "./icons/calendarIcon.svg?react"; import CaretDownIcon from "./icons/caretDownIcon.svg?react"; +import CastIcon from "./icons/castIcon.svg?react"; import CertifiedIcon from "./icons/certifiedIcon.svg?react"; -import CheckCircleIcon from "./icons/checkCircleIcon.svg?react"; +import ChartIcon from "./icons/chartIcon.svg?react"; +import ChatIcon from "./icons/chatIcon.svg?react"; import CheckIcon from "./icons/checkIcon.svg?react"; import ChevronDownIcon from "./icons/chevronDownIcon.svg?react"; import ChevronLeftIcon from "./icons/chevronLeftIcon.svg?react"; import ChevronRightIcon from "./icons/chevronRightIcon.svg?react"; import ChevronUpIcon from "./icons/chevronUpIcon.svg?react"; -import ClockIcon from "./icons/clockIcon.svg?react"; +import ClockFilledIcon from "./icons/clockFilledIcon.svg?react"; import ClockOutlineIcon from "./icons/clockOutlineIcon.svg?react"; +import CommentsIcon from "./icons/commentsIcon.svg?react"; +import CommunityIcon from "./icons/communityIcon.svg?react"; +import ConnectionIcon from "./icons/connectionIcon.svg?react"; +import ContractIcon from "./icons/contractIcon.svg?react"; import CopyIcon from "./icons/copyIcon.svg?react"; import CreditsIcon from "./icons/creditsIcon.svg?react"; import CrossIcon from "./icons/crossIcon.svg?react"; +import DatabaseIcon from "./icons/databaseIcon.svg?react"; import DayIcon from "./icons/dayIcon.svg?react"; +import DbtCloudIcon from "./icons/dbtCloudIcon.svg?react"; import DestinationIcon from "./icons/destinationIcon.svg?react"; import DisabledIcon from "./icons/disabledIcon.svg?react"; +import DockerIcon from "./icons/dockerIcon.svg?react"; import DocsIcon from "./icons/docsIcon.svg?react"; import DownloadIcon from "./icons/downloadIcon.svg?react"; import DuplicateIcon from "./icons/duplicateIcon.svg?react"; import EarthIcon from "./icons/earthIcon.svg?react"; -import ErrorIcon from "./icons/errorIcon.svg?react"; +import EqualIcon from "./icons/equalIcon.svg?react"; +import ErrorFilledIcon from "./icons/errorFilledIcon.svg?react"; +import ErrorOutlineIcon from "./icons/errorOutlineIcon.svg?react"; import ExpandIcon from "./icons/expandIcon.svg?react"; import EyeIcon from "./icons/eyeIcon.svg?react"; +import EyeSlashIcon from "./icons/eyeSlashIcon.svg?react"; import FileIcon from "./icons/fileIcon.svg?react"; +import FilesIcon from "./icons/filesIcon.svg?react"; import FlashIcon from "./icons/flashIcon.svg?react"; import FolderIcon from "./icons/folderIcon.svg?react"; import GearIcon from "./icons/gearIcon.svg?react"; import GlobeIcon from "./icons/globeIcon.svg?react"; +import GoogleIcon from "./icons/googleIcon.svg?react"; import GridIcon from "./icons/gridIcon.svg?react"; +import HelpIcon from "./icons/helpIcon.svg?react"; +import HouseIcon from "./icons/houseIcon.svg?react"; +import IdCardIcon from "./icons/idCardIcon.svg?react"; import ImportIcon from "./icons/importIcon.svg?react"; -import InfoIcon from "./icons/infoIcon.svg?react"; +import InfoFilledIcon from "./icons/infoFilledIcon.svg?react"; +import InfoOutlineIcon from "./icons/infoOutlineIcon.svg?react"; +import IntegrationsIcon from "./icons/integrationsIcon.svg?react"; +import KeyCircleIcon from "./icons/keyCircleIcon.svg?react"; +import LayersIcon from "./icons/layersIcon.svg?react"; import LensIcon from "./icons/lensIcon.svg?react"; +import LightbulbIcon from "./icons/lightbulbIcon.svg?react"; +import LinkIcon from "./icons/linkIcon.svg?react"; +import LoadingIcon from "./icons/loadingIcon.svg?react"; import LocationIcon from "./icons/locationIcon.svg?react"; -import LockedIcon from "./icons/lockedIcon.svg?react"; +import LockIcon from "./icons/lockIcon.svg?react"; +import MenuIcon from "./icons/menuIcon.svg?react"; +import MinusCircleIcon from "./icons/minusCircleIcon.svg?react"; import MinusIcon from "./icons/minusIcon.svg?react"; import ModificationIcon from "./icons/modificationIcon.svg?react"; +import MonitorIcon from "./icons/monitorIcon.svg?react"; import MoonIcon from "./icons/moonIcon.svg?react"; -import MoveHandleIcon from "./icons/moveHandleIcon.svg?react"; import NestedIcon from "./icons/nestedIcon.svg?react"; import NoteIcon from "./icons/noteIcon.svg?react"; import NotificationIcon from "./icons/notificationIcon.svg?react"; +import OnboardingIcon from "./icons/onboardingIcon.svg?react"; import OptionsIcon from "./icons/optionsIcon.svg?react"; import ParametersIcon from "./icons/parametersIcon.svg?react"; -import PauseIcon from "./icons/pauseIcon.svg?react"; +import PauseFilledIcon from "./icons/pauseFilledIcon.svg?react"; import PauseOutlineIcon from "./icons/pauseOutlineIcon.svg?react"; import PencilIcon from "./icons/pencilIcon.svg?react"; import PlayIcon from "./icons/playIcon.svg?react"; import PlusIcon from "./icons/plusIcon.svg?react"; -import PodcastIcon from "./icons/podcastIcon.svg?react"; import PrefixIcon from "./icons/prefixIcon.svg?react"; -import ReduceIcon from "./icons/reduceIcon.svg?react"; +import PulseIcon from "./icons/pulseIcon.svg?react"; +import QuestionIcon from "./icons/questionIcon.svg?react"; +import RecipesIcon from "./icons/recipesIcon.svg?react"; import ResetIcon from "./icons/resetIcon.svg?react"; +import RocketIcon from "./icons/rocketIcon.svg?react"; import RotateIcon from "./icons/rotateIcon.svg?react"; +import SchemaIcon from "./icons/schemaIcon.svg?react"; +import SelectIcon from "./icons/selectIcon.svg?react"; import ShareIcon from "./icons/shareIcon.svg?react"; +import ShortVideoIcon from "./icons/shortVideoIcon.svg?react"; import ShrinkIcon from "./icons/shrinkIcon.svg?react"; +import SimpleCircleIcon from "./icons/simpleCircleIcon.svg?react"; +import SlackIcon from "./icons/slackIcon.svg?react"; import SleepIcon from "./icons/sleepIcon.svg?react"; import SourceIcon from "./icons/sourceIcon.svg?react"; import StarIcon from "./icons/starIcon.svg?react"; -import StopIcon from "./icons/stopIcon.svg?react"; +import StarsIcon from "./icons/starsIcon.svg?react"; +import StatusCancelledIcon from "./icons/statusCancelledIcon.svg?react"; +import StatusErrorIcon from "./icons/statusErrorIcon.svg?react"; +import StatusInactiveIcon from "./icons/statusInactiveIcon.svg?react"; +import StatusInProgressIcon from "./icons/statusInProgressIcon.svg?react"; +import StatusSleepIcon from "./icons/statusSleepIcon.svg?react"; +import StatusSuccessIcon from "./icons/statusSuccessIcon.svg?react"; +import StatusWarningIcon from "./icons/statusWarningIcon.svg?react"; +import StopFilledIcon from "./icons/stopFilledIcon.svg?react"; import StopOutlineIcon from "./icons/stopOutlineIcon.svg?react"; -import SuccessIcon from "./icons/successIcon.svg?react"; +import SuccessFilledIcon from "./icons/successFilledIcon.svg?react"; import SuccessOutlineIcon from "./icons/successOutlineIcon.svg?react"; +import SuitcaseIcon from "./icons/suitcaseIcon.svg?react"; import SyncIcon from "./icons/syncIcon.svg?react"; +import TableIcon from "./icons/tableIcon.svg?react"; import TargetIcon from "./icons/targetIcon.svg?react"; +import TicketIcon from "./icons/ticketIcon.svg?react"; import TrashIcon from "./icons/trashIcon.svg?react"; -import UnlockedIcon from "./icons/unlockedIcon.svg?react"; +import UnlockIcon from "./icons/unlockIcon.svg?react"; import UserIcon from "./icons/userIcon.svg?react"; -import WarningIcon from "./icons/warningIcon.svg?react"; +import WarningFilledIcon from "./icons/warningFilledIcon.svg?react"; import WarningOutlineIcon from "./icons/warningOutlineIcon.svg?react"; +import WrenchIcon from "./icons/wrenchIcon.svg?react"; import { IconColor, IconProps, IconType } from "./types"; const colorMap: Record = { @@ -96,79 +146,128 @@ const sizeMap: Record, string> = { }; export const Icons: Record>> = { + addCircle: AddCircleIcon, arrowLeft: ArrowLeftIcon, arrowRight: ArrowRightIcon, article: ArticleIcon, + bell: BellIcon, + calendarCheck: CalendarCheckIcon, calendar: CalendarIcon, caretDown: CaretDownIcon, + cast: CastIcon, certified: CertifiedIcon, + chart: ChartIcon, + chat: ChatIcon, check: CheckIcon, - checkCircle: CheckCircleIcon, chevronDown: ChevronDownIcon, chevronLeft: ChevronLeftIcon, chevronRight: ChevronRightIcon, chevronUp: ChevronUpIcon, - clock: ClockIcon, + clockFilled: ClockFilledIcon, clockOutline: ClockOutlineIcon, + comments: CommentsIcon, + community: CommunityIcon, + connection: ConnectionIcon, + contract: ContractIcon, copy: CopyIcon, credits: CreditsIcon, cross: CrossIcon, + database: DatabaseIcon, day: DayIcon, + dbtCloud: DbtCloudIcon, destination: DestinationIcon, disabled: DisabledIcon, + docker: DockerIcon, docs: DocsIcon, download: DownloadIcon, duplicate: DuplicateIcon, earth: EarthIcon, - error: ErrorIcon, - eye: EyeIcon, + equal: EqualIcon, + errorFilled: ErrorFilledIcon, + errorOutline: ErrorOutlineIcon, expand: ExpandIcon, + eye: EyeIcon, + eyeSlash: EyeSlashIcon, file: FileIcon, + files: FilesIcon, flash: FlashIcon, folder: FolderIcon, gear: GearIcon, globe: GlobeIcon, + google: GoogleIcon, grid: GridIcon, + help: HelpIcon, + house: HouseIcon, + idCard: IdCardIcon, import: ImportIcon, - info: InfoIcon, + infoFilled: InfoFilledIcon, + infoOutline: InfoOutlineIcon, + integrations: IntegrationsIcon, + keyCircle: KeyCircleIcon, + layers: LayersIcon, lens: LensIcon, + lightbulb: LightbulbIcon, + link: LinkIcon, + loading: LoadingIcon, location: LocationIcon, - locked: LockedIcon, + lock: LockIcon, + menu: MenuIcon, + minusCircle: MinusCircleIcon, minus: MinusIcon, modification: ModificationIcon, + monitor: MonitorIcon, moon: MoonIcon, - moveHandle: MoveHandleIcon, nested: NestedIcon, note: NoteIcon, notification: NotificationIcon, + onboarding: OnboardingIcon, options: OptionsIcon, parameters: ParametersIcon, - pause: PauseIcon, + pauseFilled: PauseFilledIcon, pauseOutline: PauseOutlineIcon, pencil: PencilIcon, play: PlayIcon, plus: PlusIcon, - podcast: PodcastIcon, prefix: PrefixIcon, - reduce: ReduceIcon, + pulse: PulseIcon, + question: QuestionIcon, + recipes: RecipesIcon, reset: ResetIcon, + rocket: RocketIcon, rotate: RotateIcon, + schema: SchemaIcon, + select: SelectIcon, share: ShareIcon, + shortVideo: ShortVideoIcon, shrink: ShrinkIcon, + simpleCircle: SimpleCircleIcon, + slack: SlackIcon, sleep: SleepIcon, source: SourceIcon, star: StarIcon, - stop: StopIcon, + stars: StarsIcon, + statusCancelled: StatusCancelledIcon, + statusError: StatusErrorIcon, + statusInactive: StatusInactiveIcon, + statusInProgress: StatusInProgressIcon, + statusSleep: StatusSleepIcon, + statusSuccess: StatusSuccessIcon, + statusWarning: StatusWarningIcon, + stopFilled: StopFilledIcon, stopOutline: StopOutlineIcon, - success: SuccessIcon, + successFilled: SuccessFilledIcon, successOutline: SuccessOutlineIcon, + suitcase: SuitcaseIcon, sync: SyncIcon, + table: TableIcon, target: TargetIcon, + ticket: TicketIcon, trash: TrashIcon, - unlocked: UnlockedIcon, + unlock: UnlockIcon, user: UserIcon, - warning: WarningIcon, + warningFilled: WarningFilledIcon, warningOutline: WarningOutlineIcon, + wrench: WrenchIcon, }; export const Icon: React.FC = React.memo( @@ -183,6 +282,8 @@ export const Icon: React.FC = React.memo( return React.createElement(Icons[type], { ...props, + // @ts-expect-error data-* attributes aren't allowed outside a JSX tag + "data-icon": kebabCase(type), className: classes, }); } diff --git a/airbyte-webapp/src/components/ui/Icon/icons/addCircleIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/addCircleIcon.svg new file mode 100644 index 00000000000..a9f660bc642 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/addCircleIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/arrowLeftIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/arrowLeftIcon.svg index 06e983067ec..99c3f2d559c 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/arrowLeftIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/arrowLeftIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/arrowRightIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/arrowRightIcon.svg index 41f32106e06..8f46632e6f3 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/arrowRightIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/arrowRightIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/articleIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/articleIcon.svg index ab022d625b8..09097344116 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/articleIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/articleIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/bellIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/bellIcon.svg new file mode 100644 index 00000000000..a32b5889471 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/bellIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/calendarCheckIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/calendarCheckIcon.svg new file mode 100644 index 00000000000..4e2dff72f10 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/calendarCheckIcon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/calendarIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/calendarIcon.svg index d9ccf742d57..d68cd0271d5 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/calendarIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/calendarIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/caretDownIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/caretDownIcon.svg index 53326433efb..bfa0d0fda35 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/caretDownIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/caretDownIcon.svg @@ -1,6 +1,3 @@ - - - \ No newline at end of file + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/castIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/castIcon.svg new file mode 100644 index 00000000000..8924951ee77 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/castIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/certifiedIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/certifiedIcon.svg index db450b2bbaa..bda377013c2 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/certifiedIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/certifiedIcon.svg @@ -1,4 +1,3 @@ - - - - \ No newline at end of file + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/chartIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/chartIcon.svg new file mode 100644 index 00000000000..195f7751170 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/chartIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/chatIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/chatIcon.svg new file mode 100644 index 00000000000..0689431a070 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/chatIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/checkCircleIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/checkCircleIcon.svg deleted file mode 100644 index 2e2e53d7dd9..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/checkCircleIcon.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - \ No newline at end of file diff --git a/airbyte-webapp/src/components/ui/Icon/icons/checkIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/checkIcon.svg index 9cdd91c0532..0f29725dc60 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/checkIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/checkIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/chevronDownIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/chevronDownIcon.svg index 4f6b6727d20..d516dcdc588 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/chevronDownIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/chevronDownIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/chevronLeftIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/chevronLeftIcon.svg index b88f7ed37de..abff02f3f20 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/chevronLeftIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/chevronLeftIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/chevronRightIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/chevronRightIcon.svg index aaf07de1a83..e3775cc94ee 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/chevronRightIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/chevronRightIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/chevronUpIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/chevronUpIcon.svg index 446773fd688..208c9294a7a 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/chevronUpIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/chevronUpIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/clockFilledIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/clockFilledIcon.svg new file mode 100644 index 00000000000..aff677b717f --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/clockFilledIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/clockIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/clockIcon.svg deleted file mode 100644 index 05e8fbc1574..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/clockIcon.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/airbyte-webapp/src/components/ui/Icon/icons/clockOutlineIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/clockOutlineIcon.svg index b63f8d6ba68..9334f2119ec 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/clockOutlineIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/clockOutlineIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/commentsIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/commentsIcon.svg new file mode 100644 index 00000000000..5b60572bb6a --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/commentsIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/communityIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/communityIcon.svg new file mode 100644 index 00000000000..6921422e1cc --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/communityIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/connectionIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/connectionIcon.svg new file mode 100644 index 00000000000..f06dc598a64 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/connectionIcon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/contractIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/contractIcon.svg new file mode 100644 index 00000000000..8546719eed6 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/contractIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/copyIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/copyIcon.svg index fe03bfb52c6..1d5d2d300cc 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/copyIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/copyIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/creditsIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/creditsIcon.svg index 01cacbdd0f9..3ea55d2a8cb 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/creditsIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/creditsIcon.svg @@ -1,4 +1,5 @@ - - - - \ No newline at end of file + + + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/crossIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/crossIcon.svg index ad84c4442b1..024653332de 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/crossIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/crossIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/databaseIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/databaseIcon.svg new file mode 100644 index 00000000000..4738d80f0bc --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/databaseIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/dayIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/dayIcon.svg index ed1e4a1f6cf..921ac3873c5 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/dayIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/dayIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/dbtCloudIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/dbtCloudIcon.svg new file mode 100644 index 00000000000..74990145d3f --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/dbtCloudIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/destinationIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/destinationIcon.svg index 5dd46563c3c..9c0e380a116 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/destinationIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/destinationIcon.svg @@ -1,3 +1,4 @@ - - + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/disabledIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/disabledIcon.svg index 787f9c0e72a..2d0054e8bf6 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/disabledIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/disabledIcon.svg @@ -1,4 +1,3 @@ - - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/dockerIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/dockerIcon.svg new file mode 100644 index 00000000000..55b4f492501 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/dockerIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/docsIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/docsIcon.svg index 15c1caa2591..2ba3bfbe11b 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/docsIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/docsIcon.svg @@ -1,3 +1,3 @@ - - - \ No newline at end of file + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/downloadIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/downloadIcon.svg index 14b9a370e3d..62388673b6b 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/downloadIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/downloadIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/earthIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/earthIcon.svg index 600a9f6fa74..9768e329a18 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/earthIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/earthIcon.svg @@ -1,3 +1,3 @@ - + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/equalIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/equalIcon.svg new file mode 100644 index 00000000000..f5b9b4c0665 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/equalIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/errorIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/errorFilledIcon.svg similarity index 100% rename from airbyte-webapp/src/components/ui/Icon/icons/errorIcon.svg rename to airbyte-webapp/src/components/ui/Icon/icons/errorFilledIcon.svg diff --git a/airbyte-webapp/src/components/ui/Icon/icons/errorOutlineIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/errorOutlineIcon.svg new file mode 100644 index 00000000000..d7265cea4df --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/errorOutlineIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/expandIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/expandIcon.svg index e56e39462f5..8d5429d4fa7 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/expandIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/expandIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/eyeIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/eyeIcon.svg index 64a0a5c1ae4..3611cdc14df 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/eyeIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/eyeIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/eyeSlashIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/eyeSlashIcon.svg new file mode 100644 index 00000000000..90f156a0784 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/eyeSlashIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/fileIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/fileIcon.svg index ea4b03e2501..728f77b63be 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/fileIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/fileIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/filesIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/filesIcon.svg new file mode 100644 index 00000000000..da25baa00cc --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/filesIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/flashIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/flashIcon.svg index 71910b033ce..2defe2023a8 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/flashIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/flashIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/folderIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/folderIcon.svg index 845c129467b..2635d1ee79e 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/folderIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/folderIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/gearIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/gearIcon.svg index cd9fa9a9098..e09c7f9892c 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/gearIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/gearIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/globeIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/globeIcon.svg index 0be8da0eb4c..7bc25c3cbff 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/globeIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/globeIcon.svg @@ -1,10 +1,3 @@ - - - - - - - - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/googleIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/googleIcon.svg new file mode 100644 index 00000000000..933fed3df72 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/googleIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/gridIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/gridIcon.svg index 968e118b564..fa3f873982c 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/gridIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/gridIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/helpIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/helpIcon.svg new file mode 100644 index 00000000000..f2e3a9072d1 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/helpIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/houseIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/houseIcon.svg new file mode 100644 index 00000000000..29a9fefff80 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/houseIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/idCardIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/idCardIcon.svg new file mode 100644 index 00000000000..d5fcdb121da --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/idCardIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/importIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/importIcon.svg index bf202176700..79436cb1f06 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/importIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/importIcon.svg @@ -1,4 +1,4 @@ - - - + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/infoFilledIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/infoFilledIcon.svg new file mode 100644 index 00000000000..3330d1df075 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/infoFilledIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/infoOutlineIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/infoOutlineIcon.svg new file mode 100644 index 00000000000..670d00c751c --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/infoOutlineIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/integrationsIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/integrationsIcon.svg new file mode 100644 index 00000000000..66691768510 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/integrationsIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/keyCircleIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/keyCircleIcon.svg new file mode 100644 index 00000000000..87ced1a4a82 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/keyCircleIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/layersIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/layersIcon.svg new file mode 100644 index 00000000000..e084268381f --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/layersIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/lensIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/lensIcon.svg index 1f058238be1..99488bfdac4 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/lensIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/lensIcon.svg @@ -1,10 +1,3 @@ - - - - - - - - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/lightbulbIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/lightbulbIcon.svg new file mode 100644 index 00000000000..e0a523442a4 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/lightbulbIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/linkIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/linkIcon.svg new file mode 100644 index 00000000000..73f05851b06 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/linkIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/loadingIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/loadingIcon.svg new file mode 100644 index 00000000000..bc6bcaaa6fc --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/loadingIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/locationIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/locationIcon.svg index 077fc39e15a..67346edb5c7 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/locationIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/locationIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/lockIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/lockIcon.svg new file mode 100644 index 00000000000..801712b7367 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/lockIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/lockedIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/lockedIcon.svg deleted file mode 100644 index b915f9df478..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/lockedIcon.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/airbyte-webapp/src/components/ui/Icon/icons/menuIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/menuIcon.svg new file mode 100644 index 00000000000..a5a04737741 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/menuIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/minusCircleIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/minusCircleIcon.svg new file mode 100644 index 00000000000..34cdb4da0ac --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/minusCircleIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/minusIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/minusIcon.svg index 4bd8c6c6298..15f47687e72 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/minusIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/minusIcon.svg @@ -1,3 +1,3 @@ - - - \ No newline at end of file + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/modificationIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/modificationIcon.svg index 4b66d317da9..845e6d480d0 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/modificationIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/modificationIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/monitorIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/monitorIcon.svg new file mode 100644 index 00000000000..144954a96ad --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/monitorIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/moonIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/moonIcon.svg index 671f425a402..1d673f12a67 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/moonIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/moonIcon.svg @@ -1,3 +1,3 @@ - - - \ No newline at end of file + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/moveHandleIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/moveHandleIcon.svg deleted file mode 100644 index 012d9558341..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/moveHandleIcon.svg +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/airbyte-webapp/src/components/ui/Icon/icons/nestedIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/nestedIcon.svg index 20c80b78644..88a1cff8e24 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/nestedIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/nestedIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/noteIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/noteIcon.svg index 726dfdcfa14..4f4ef1ea6da 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/noteIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/noteIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/notificationIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/notificationIcon.svg index a67bc243586..f35842b2625 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/notificationIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/notificationIcon.svg @@ -1,10 +1,3 @@ - - - - - - - - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/onboardingIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/onboardingIcon.svg new file mode 100644 index 00000000000..7448345b578 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/onboardingIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/optionsIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/optionsIcon.svg index 673b8b3c658..83c70422550 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/optionsIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/optionsIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/parametersIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/parametersIcon.svg index d5d5ec017ed..b8008bcdc4e 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/parametersIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/parametersIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/pauseIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/pauseFilledIcon.svg similarity index 100% rename from airbyte-webapp/src/components/ui/Icon/icons/pauseIcon.svg rename to airbyte-webapp/src/components/ui/Icon/icons/pauseFilledIcon.svg diff --git a/airbyte-webapp/src/components/ui/Icon/icons/pauseOutlineIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/pauseOutlineIcon.svg index 6af7fa27781..b6b08256dd9 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/pauseOutlineIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/pauseOutlineIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/pencilIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/pencilIcon.svg index eff4ab93b2f..b937d8190a6 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/pencilIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/pencilIcon.svg @@ -1,3 +1,3 @@ - - - \ No newline at end of file + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/play2Icon.svg b/airbyte-webapp/src/components/ui/Icon/icons/play2Icon.svg deleted file mode 100644 index a1640fc3bdc..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/play2Icon.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/airbyte-webapp/src/components/ui/Icon/icons/playIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/playIcon.svg index 015813a5c68..967869556e0 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/playIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/playIcon.svg @@ -1,3 +1,3 @@ - - - \ No newline at end of file + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/plusIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/plusIcon.svg index 7852aaba82a..2f971b4fabd 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/plusIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/plusIcon.svg @@ -1,3 +1,3 @@ - - - \ No newline at end of file + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/podcastIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/podcastIcon.svg deleted file mode 100644 index a95b5ce19d5..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/podcastIcon.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/airbyte-webapp/src/components/ui/Icon/icons/prefixIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/prefixIcon.svg index 6f7acf1a615..7688c99baa2 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/prefixIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/prefixIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/pulseIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/pulseIcon.svg new file mode 100644 index 00000000000..fefd6fee5ff --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/pulseIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/questionIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/questionIcon.svg new file mode 100644 index 00000000000..77070902a33 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/questionIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/recipesIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/recipesIcon.svg new file mode 100644 index 00000000000..68306459ae7 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/recipesIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/reduceIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/reduceIcon.svg deleted file mode 100644 index 6bf192c995c..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/reduceIcon.svg +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/airbyte-webapp/src/components/ui/Icon/icons/resetIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/resetIcon.svg index 38a8278ee5d..7bd47c27304 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/resetIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/resetIcon.svg @@ -1,3 +1,3 @@ - - - + + + \ No newline at end of file diff --git a/airbyte-webapp/src/components/ui/Icon/icons/rocketIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/rocketIcon.svg new file mode 100644 index 00000000000..bc0e97f85c7 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/rocketIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/rotateIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/rotateIcon.svg index bfb99d6ede1..a2f27c86d7a 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/rotateIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/rotateIcon.svg @@ -1,3 +1,3 @@ - - + + \ No newline at end of file diff --git a/airbyte-webapp/src/components/ui/Icon/icons/schemaIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/schemaIcon.svg new file mode 100644 index 00000000000..1341cfe5532 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/schemaIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/selectIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/selectIcon.svg new file mode 100644 index 00000000000..7a5e3252bb0 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/selectIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/shareIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/shareIcon.svg index 557af98245d..32d3d0bc321 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/shareIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/shareIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/shortVideoIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/shortVideoIcon.svg new file mode 100644 index 00000000000..6120846461a --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/shortVideoIcon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/shrinkIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/shrinkIcon.svg index a12ebcb952a..17ce4e946ef 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/shrinkIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/shrinkIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/simpleCircleIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/simpleCircleIcon.svg new file mode 100644 index 00000000000..352319042ae --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/simpleCircleIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/slackIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/slackIcon.svg new file mode 100644 index 00000000000..14a9de91bde --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/slackIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/sleepIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/sleepIcon.svg index dec8cc6955d..e3422ac4312 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/sleepIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/sleepIcon.svg @@ -1,3 +1,4 @@ - - - \ No newline at end of file + + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/sourceIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/sourceIcon.svg index fc5ccdaba22..4fb445c8cea 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/sourceIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/sourceIcon.svg @@ -1,3 +1,4 @@ - - + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/starIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/starIcon.svg index fe646a6da60..a445a545ef4 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/starIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/starIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/starsIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/starsIcon.svg new file mode 100644 index 00000000000..c384d43d860 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/starsIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/statusCancelledIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/statusCancelledIcon.svg new file mode 100644 index 00000000000..7978ae79a3f --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/statusCancelledIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/statusErrorIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/statusErrorIcon.svg new file mode 100644 index 00000000000..10ad01e76f3 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/statusErrorIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/statusInProgressIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/statusInProgressIcon.svg new file mode 100644 index 00000000000..0e649db6bc9 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/statusInProgressIcon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/statusInactiveIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/statusInactiveIcon.svg new file mode 100644 index 00000000000..70eb4281ba4 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/statusInactiveIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/statusSleepIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/statusSleepIcon.svg new file mode 100644 index 00000000000..434d727d495 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/statusSleepIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/statusSuccessIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/statusSuccessIcon.svg new file mode 100644 index 00000000000..ea95133f243 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/statusSuccessIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/statusWarningIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/statusWarningIcon.svg new file mode 100644 index 00000000000..6450e18c333 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/statusWarningIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/stopFilledIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/stopFilledIcon.svg new file mode 100644 index 00000000000..d8cdd9abd9c --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/stopFilledIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/stopIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/stopIcon.svg deleted file mode 100644 index ba5c78c9c07..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/stopIcon.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/airbyte-webapp/src/components/ui/Icon/icons/stopOutlineIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/stopOutlineIcon.svg index 6fa49b7326f..9fe3ae84cf6 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/stopOutlineIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/stopOutlineIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/successIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/successFilledIcon.svg similarity index 100% rename from airbyte-webapp/src/components/ui/Icon/icons/successIcon.svg rename to airbyte-webapp/src/components/ui/Icon/icons/successFilledIcon.svg diff --git a/airbyte-webapp/src/components/ui/Icon/icons/successOutlineIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/successOutlineIcon.svg index 95d297da5c4..b6353bd2535 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/successOutlineIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/successOutlineIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/suitcaseIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/suitcaseIcon.svg new file mode 100644 index 00000000000..8f8f1161e4e --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/suitcaseIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/syncIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/syncIcon.svg index 7dd56ce900a..e5cac8203f8 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/syncIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/syncIcon.svg @@ -1,10 +1,3 @@ - - - - - - - - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/tableIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/tableIcon.svg new file mode 100644 index 00000000000..296e42d8b0f --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/tableIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/targetIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/targetIcon.svg index d43742c2f36..f729cb9b507 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/targetIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/targetIcon.svg @@ -1,7 +1,8 @@ - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/ticketIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/ticketIcon.svg new file mode 100644 index 00000000000..5ff5143127f --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/ticketIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/trashIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/trashIcon.svg index bd47985598a..01c0d095fc2 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/trashIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/trashIcon.svg @@ -1,10 +1,3 @@ - - - - - - - - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/unlockIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/unlockIcon.svg new file mode 100644 index 00000000000..4cb245362e7 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/unlockIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/unlockedIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/unlockedIcon.svg deleted file mode 100644 index 5997e8ccd6e..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/unlockedIcon.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/airbyte-webapp/src/components/ui/Icon/icons/userIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/userIcon.svg index bfde8739855..4a477dcea0d 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/userIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/userIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/warningFilledIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/warningFilledIcon.svg new file mode 100644 index 00000000000..da80caed8f3 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/warningFilledIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/warningIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/warningIcon.svg deleted file mode 100644 index 96bd940c193..00000000000 --- a/airbyte-webapp/src/components/ui/Icon/icons/warningIcon.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/airbyte-webapp/src/components/ui/Icon/icons/warningOutlineIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/warningOutlineIcon.svg index 5d1d8312ce5..6ebdfc92bd9 100644 --- a/airbyte-webapp/src/components/ui/Icon/icons/warningOutlineIcon.svg +++ b/airbyte-webapp/src/components/ui/Icon/icons/warningOutlineIcon.svg @@ -1,3 +1,3 @@ - - + + diff --git a/airbyte-webapp/src/components/ui/Icon/icons/wrenchIcon.svg b/airbyte-webapp/src/components/ui/Icon/icons/wrenchIcon.svg new file mode 100644 index 00000000000..28a4a84388e --- /dev/null +++ b/airbyte-webapp/src/components/ui/Icon/icons/wrenchIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/components/ui/Icon/types.ts b/airbyte-webapp/src/components/ui/Icon/types.ts index f89858016b2..bf0c0958219 100644 --- a/airbyte-webapp/src/components/ui/Icon/types.ts +++ b/airbyte-webapp/src/components/ui/Icon/types.ts @@ -1,77 +1,126 @@ export type IconType = + | "addCircle" + | "arrowLeft" + | "arrowRight" | "article" + | "bell" + | "calendarCheck" | "calendar" | "caretDown" + | "cast" | "certified" + | "chart" + | "chat" | "check" - | "checkCircle" | "chevronDown" | "chevronLeft" | "chevronRight" | "chevronUp" - | "clock" + | "clockFilled" | "clockOutline" + | "comments" + | "community" + | "connection" + | "contract" | "copy" | "credits" | "cross" + | "database" | "day" + | "dbtCloud" | "destination" | "disabled" + | "docker" | "docs" | "download" | "duplicate" | "earth" - | "error" + | "equal" + | "errorFilled" + | "errorOutline" | "expand" | "eye" + | "eyeSlash" | "file" + | "files" | "flash" | "folder" | "gear" | "globe" + | "google" | "grid" + | "help" + | "house" + | "idCard" | "import" - | "info" - | "arrowLeft" + | "infoFilled" + | "infoOutline" + | "integrations" + | "keyCircle" + | "layers" | "lens" + | "lightbulb" + | "link" + | "loading" | "location" - | "locked" + | "lock" + | "menu" + | "minusCircle" | "minus" | "modification" + | "monitor" | "moon" - | "moveHandle" | "nested" | "note" | "notification" + | "onboarding" | "options" | "parameters" - | "pause" + | "pauseFilled" | "pauseOutline" | "pencil" | "play" | "plus" - | "podcast" | "prefix" - | "reduce" + | "pulse" + | "question" + | "recipes" | "reset" - | "arrowRight" + | "rocket" | "rotate" + | "schema" + | "select" | "share" + | "shortVideo" | "shrink" + | "simpleCircle" + | "slack" | "sleep" | "source" | "star" - | "stop" + | "stars" + | "statusCancelled" + | "statusError" + | "statusInactive" + | "statusInProgress" + | "statusSleep" + | "statusSuccess" + | "statusWarning" + | "stopFilled" | "stopOutline" - | "success" + | "successFilled" | "successOutline" + | "suitcase" | "sync" + | "table" | "target" + | "ticket" | "trash" - | "unlocked" + | "unlock" | "user" - | "warning" - | "warningOutline"; + | "warningFilled" + | "warningOutline" + | "wrench"; export type IconColor = "primary" | "disabled" | "action" | "success" | "error" | "warning" | "affordance"; diff --git a/airbyte-webapp/src/components/ui/Input/Input.module.scss b/airbyte-webapp/src/components/ui/Input/Input.module.scss index 195f986301b..1bd4a17547f 100644 --- a/airbyte-webapp/src/components/ui/Input/Input.module.scss +++ b/airbyte-webapp/src/components/ui/Input/Input.module.scss @@ -82,7 +82,7 @@ button.visibilityButton { top: 0; display: flex; height: 100%; - width: 30px; + width: 40px; align-items: center; justify-content: center; border: none; diff --git a/airbyte-webapp/src/components/ui/Input/Input.test.tsx b/airbyte-webapp/src/components/ui/Input/Input.test.tsx index 6dfe5c2f3dd..864e0e9bec1 100644 --- a/airbyte-webapp/src/components/ui/Input/Input.test.tsx +++ b/airbyte-webapp/src/components/ui/Input/Input.test.tsx @@ -30,16 +30,16 @@ describe("", () => { it("renders password input with visibilty button", async () => { const value = "eight888"; - const { getByTestId, getByRole } = await render(); + const { getByTestId } = await render(); expect(getByTestId("input")).toHaveAttribute("type", "password"); expect(getByTestId("input")).toHaveValue(value); - expect(getByRole("img", { hidden: true })).toHaveAttribute("data-icon", "eye"); + expect(getByTestId("mocksvg")).toHaveAttribute("data-icon", "eye"); }); it("renders visible password when visibility button is clicked", async () => { const value = "eight888"; - const { getByTestId, getByRole } = await render(); + const { getByTestId } = await render(); await userEvent.click(getByTestId("toggle-password-visibility-button")); @@ -48,7 +48,7 @@ describe("", () => { expect(inputEl).toHaveAttribute("type", "text"); expect(inputEl).toHaveValue(value); expect(inputEl.selectionStart).toBe(value.length); - expect(getByRole("img", { hidden: true })).toHaveAttribute("data-icon", "eye-slash"); + expect(getByTestId("mocksvg")).toHaveAttribute("data-icon", "eye-slash"); }); it("showing password should remember cursor position", async () => { @@ -69,7 +69,7 @@ describe("", () => { it("hides password on blur", async () => { const value = "eight888"; - const { getByTestId, getByRole } = await render(); + const { getByTestId } = await render(); getByTestId("toggle-password-visibility-button").click(); @@ -80,7 +80,7 @@ describe("", () => { await waitFor(() => { expect(inputEl).toHaveAttribute("type", "password"); - expect(getByRole("img", { hidden: true })).toHaveAttribute("data-icon", "eye"); + expect(getByTestId("mocksvg")).toHaveAttribute("data-icon", "eye"); }); }); diff --git a/airbyte-webapp/src/components/ui/Input/Input.tsx b/airbyte-webapp/src/components/ui/Input/Input.tsx index 862fc7791bd..e6833e31a57 100644 --- a/airbyte-webapp/src/components/ui/Input/Input.tsx +++ b/airbyte-webapp/src/components/ui/Input/Input.tsx @@ -1,10 +1,10 @@ -import { faEye, faEyeSlash } from "@fortawesome/free-regular-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import React, { ReactNode, useCallback, useImperativeHandle, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { useToggle } from "react-use"; +import { Icon } from "components/ui/Icon"; + import styles from "./Input.module.scss"; import { Button } from "../Button"; @@ -115,14 +115,13 @@ export const Input = React.forwardRef( focusOnInputElement(); }} tabIndex={-1} - size="xs" type="button" variant="clear" aria-label={formatMessage({ id: `ui.input.${isContentVisible ? "hide" : "show"}Password`, })} data-testid="toggle-password-visibility-button" - icon={} + icon={} /> ) : null}
diff --git a/airbyte-webapp/src/components/ui/ListBox/ListBox.stories.tsx b/airbyte-webapp/src/components/ui/ListBox/ListBox.stories.tsx index f8d3ff16fe4..026a3364ff9 100644 --- a/airbyte-webapp/src/components/ui/ListBox/ListBox.stories.tsx +++ b/airbyte-webapp/src/components/ui/ListBox/ListBox.stories.tsx @@ -1,9 +1,8 @@ -import { faEdit } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { Meta, StoryFn } from "@storybook/react"; import { useState } from "react"; import { FlexContainer } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { ListBox, ListBoxProps } from "./ListBox"; @@ -40,7 +39,7 @@ const options = [ { label: "one", value: 1, - icon: , + icon: , }, { label: "two", diff --git a/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx b/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx index f8794649b52..3c9d9a60e8e 100644 --- a/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx +++ b/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx @@ -35,7 +35,7 @@ const DefaultControlButton = ({ selectedOption, isDisabled }: ListBoxControl )} - + ); }; @@ -45,7 +45,7 @@ export interface Option { value: T; icon?: React.ReactNode; disabled?: boolean; - testId?: string; + "data-testid"?: string; } export interface ListBoxProps { @@ -102,7 +102,12 @@ export const ListBox = ({ }; return ( -
+
({ e.stopPropagation()} + {...(testId && { + "data-testid": `${testId}-listbox-button`, + })} > {options.length > 0 && ( <> - {options.map(({ label, value, icon, disabled, testId }, index) => ( + {options.map(({ label, value, icon, disabled, ...restOptionProps }, index) => ( ({ [styles.disabled]: disabled, })} onClick={(e) => e.stopPropagation()} - {...(testId && { "data-testid": testId })} + {...(restOptionProps["data-testid"] && { + "data-testid": `${restOptionProps["data-testid"]}-option`, + })} > {({ active, selected }) => ( > = { + warning: "statusWarning", + error: "statusError", + success: "statusSuccess", + info: "infoFilled", }; const STYLES_BY_TYPE: Readonly> = { @@ -77,10 +75,7 @@ export const Message: React.FC> = ({ > {!hideIcon && (
- +
)}
@@ -109,7 +104,7 @@ export const Message: React.FC> = ({ className={styles.closeButton} onClick={onClose} size="xs" - icon={} + icon={} /> )}
diff --git a/airbyte-webapp/src/components/ui/Multiselect/Multiselect.module.scss b/airbyte-webapp/src/components/ui/Multiselect/Multiselect.module.scss new file mode 100644 index 00000000000..93298e7f4af --- /dev/null +++ b/airbyte-webapp/src/components/ui/Multiselect/Multiselect.module.scss @@ -0,0 +1,138 @@ +@use "scss/colors"; +@use "scss/variables"; + +.container { + box-shadow: none; + padding: 0; + margin: 0; + + & :global(.rw-multiselect) { + & :global(.rw-list-option.rw-state-focus), + & :global(.rw-list-option) { + color: colors.$dark-blue; + border: none; + padding: 10px 16px; + font-size: 14px; + line-height: 19px; + } + + & :global(.rw-list-option):hover { + background: colors.$grey-50; + } + + & :global(.rw-list-option.rw-state-selected) { + background: colors.$blue-50; + color: colors.$blue; + pointer-events: none; + } + + & :global(.rw-popup) { + border: 0.5px solid colors.$grey-100; + border-radius: 4px; + box-shadow: variables.$box-shadow-menu; + background-color: colors.$foreground; + } + + & :global(.rw-popup-container) { + & :global(.rw-select) { + display: none; + } + + & :global(.rw-list-optgroup) { + width: 100%; + padding: 0; + border: none; + } + } + + &.error :global(.rw-widget-container) { + border-color: colors.$red; + + &:hover:not(:global(.rw-state-readonly), :global(.rw-state-disabled)) { + border-color: colors.$red; + } + } + + & :global(.rw-widget-container) { + box-shadow: none; + outline: none; + width: 100%; + min-height: 37px; + padding: 4px 8px; + border-radius: 4px; + font-size: 14px; + line-height: 20px; + font-weight: normal; + border: 1px solid colors.$grey-50; + background: colors.$grey-50; + caret-color: colors.$dark-blue; + + & > div { + vertical-align: middle; + } + + & :global(.rw-btn-select) { + color: colors.$blue; + } + + & input { + padding: 0; + height: auto; + line-height: 26px; + color: colors.$dark-blue; + } + + &::placeholder { + color: colors.$grey-300; + } + + &:hover:not(:global(.rw-state-readonly), :global(.rw-state-disabled)) { + box-shadow: none; + border-color: colors.$grey-100; + background-color: colors.$grey-50; + } + + & :global(.rw-multiselect-taglist) { + vertical-align: top; + + & :global(.rw-multiselect-tag) { + background-color: colors.$grey-100; + border-color: colors.$grey; + color: colors.$grey-600; + border-radius: 4px; + font-weight: 500; + font-size: 12px; + line-height: 21px; + height: 23px; + margin: 0 3px 0 0; + padding: 0 4px 0 6px; + + & :global(.rw-multiselect-tag-btn) { + color: colors.$grey-400; + font-size: 20px; + line-height: 23px; + } + } + } + } + + &:global(.rw-state-focus):not(:global(.rw-state-disabled), :global(.rw-state-readonly)) { + & :global(.rw-widget-container) { + border-color: colors.$blue; + } + } + + &:global(.rw-state-disabled), + &:global(.rw-state-readonly) { + cursor: not-allowed; + + & :global(.rw-widget-container) { + pointer-events: none; + color: colors.$grey-400; + border: none; + box-shadow: none; + background-color: colors.$grey-50; + } + } + } +} diff --git a/airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx b/airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx index 3b4cc0e6b7c..f76ceb3b8ce 100644 --- a/airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx +++ b/airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx @@ -1,6 +1,8 @@ +import classNames from "classnames"; import { Multiselect as ReactMultiselect } from "react-widgets"; import { MultiselectProps as WidgetMultiselectProps } from "react-widgets/lib/Multiselect"; -import styled from "styled-components"; + +import styles from "./Multiselect.module.scss"; import "react-widgets/dist/css/react-widgets.css"; export interface MultiselectProps extends WidgetMultiselectProps { @@ -9,130 +11,10 @@ export interface MultiselectProps extends WidgetMultiselectProps { name?: string; } -export const Multiselect = styled(ReactMultiselect)` - box-shadow: none; - padding: 0; - margin: 0; - - & .rw-list-option.rw-state-focus, - & .rw-list-option { - color: ${({ theme }) => theme.textColor}; - border: none; - padding: 10px 16px; - font-size: 14px; - line-height: 19px; - } - - & .rw-list-option:hover { - background: ${({ theme }) => theme.primaryColor12}; - color: ${({ theme }) => theme.primaryColor}; - } - - & .rw-list-option.rw-state-selected { - background: ${({ theme }) => theme.primaryColor12}; - color: ${({ theme }) => theme.primaryColor}; - pointer-events: none; - } - - & .rw-popup { - border: 0.5px solid ${({ theme }) => theme.greyColor20}; - border-radius: 4px; - box-shadow: - 0 8px 10px 0 rgba(11, 10, 26, 0.04), - 0 3px 14px 0 rgba(11, 10, 26, 0.08), - 0 5px 5px 0 rgba(11, 10, 26, 0.12); - } - - & .rw-popup-container { - & .rw-select { - display: none; - } - - & .rw-list-optgroup { - width: 100%; - padding: 0; - border: none; - } - } - - & .rw-widget-container { - box-shadow: none; - outline: none; - width: 100%; - min-height: 37px; - padding: 4px 8px; - border-radius: 4px; - font-size: 14px; - line-height: 20px; - font-weight: normal; - border: 1px solid ${(props) => (props.error ? props.theme.dangerColor : props.theme.greyColor0)}; - background: ${(props) => (props.error ? props.theme.greyColor10 : props.theme.greyColor0)}; - caret-color: ${({ theme }) => theme.primaryColor}; - - & > div { - vertical-align: middle; - } - - & .rw-btn-select { - color: ${({ theme }) => theme.primaryColor}; - } - - & input { - padding: 0; - height: auto; - line-height: 26px; - } - - &::placeholder { - color: ${({ theme }) => theme.greyColor40}; - } - - &:hover:not(.rw-state-readonly, .rw-state-disabled) { - box-shadow: none; - border-color: ${(props) => (props.error ? props.theme.dangerColor : props.theme.greyColor20)}; - } - - & .rw-multiselect-taglist { - vertical-align: top; - - & .rw-multiselect-tag { - background: ${({ theme }) => theme.mediumPrimaryColor}; - border-color: ${({ theme }) => theme.mediumPrimaryColor}; - color: ${({ theme }) => theme.whiteColor}; - border-radius: 4px; - font-weight: 500; - font-size: 12px; - line-height: 21px; - height: 23px; - margin: 0 3px 0 0; - padding: 0 4px 0 6px; - - & .rw-multiselect-tag-btn { - color: ${({ theme }) => theme.greyColor55}; - font-size: 20px; - line-height: 23px; - } - } - } - } - - &.rw-state-focus:not(.rw-state-disabled, .rw-state-readonly) { - & .rw-widget-container { - background: ${({ theme }) => theme.primaryColor12}; - border-color: ${({ theme }) => theme.primaryColor}; - } - } - - &.rw-state-disabled, - &.rw-state-readonly { - cursor: not-allowed; - - & .rw-widget-container { - pointer-events: none; - color: ${({ theme }) => theme.greyColor55}; - border: none; - box-shadow: none; - background-color: ${({ theme }) => theme.greyColor0}; - } - } -`; +export const Multiselect: React.FC = (props) => { + return ( +
+ +
+ ); +}; diff --git a/airbyte-webapp/src/components/ui/Pre/Pre.module.scss b/airbyte-webapp/src/components/ui/Pre/Pre.module.scss index 294d270dda4..4d3db8f84a6 100644 --- a/airbyte-webapp/src/components/ui/Pre/Pre.module.scss +++ b/airbyte-webapp/src/components/ui/Pre/Pre.module.scss @@ -4,7 +4,7 @@ .content { margin: 0; color: colors.$dark-blue; - font-size: variables.$font-size-lg; + font-size: inherit; } .longLines { diff --git a/airbyte-webapp/src/components/ui/SearchInput/SearchInput.tsx b/airbyte-webapp/src/components/ui/SearchInput/SearchInput.tsx index 994aadfe83b..c1803abd654 100644 --- a/airbyte-webapp/src/components/ui/SearchInput/SearchInput.tsx +++ b/airbyte-webapp/src/components/ui/SearchInput/SearchInput.tsx @@ -20,7 +20,7 @@ export const SearchInput: React.FC = ({ value, onChange, place // eslint-disable-next-line jsx-a11y/label-has-associated-control
), enableSorting: false, diff --git a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudHelpDropdown.tsx b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudHelpDropdown.tsx index 6b2840dd3e6..5a44db7e4e4 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudHelpDropdown.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudHelpDropdown.tsx @@ -1,17 +1,12 @@ -import { faCalendarCheck, faQuestionCircle } from "@fortawesome/free-regular-svg-icons"; -import { faUpRightFromSquare } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { FormattedMessage, useIntl } from "react-intl"; -import { DocsIcon } from "components/icons/DocsIcon"; import { DropdownMenuOptionType } from "components/ui/DropdownMenu"; +import { Icon } from "components/ui/Icon"; import { links } from "core/utils/links"; import { CloudRoutes } from "packages/cloud/cloudRoutePaths"; import { useZendesk } from "packages/cloud/services/thirdParty/zendesk"; -import ChatIcon from "views/layout/SideBar/components/ChatIcon"; import { NavDropdown } from "views/layout/SideBar/components/NavDropdown"; -import StatusIcon from "views/layout/SideBar/components/StatusIcon"; export const CloudHelpDropdown: React.FC = () => { const { formatMessage } = useIntl(); @@ -24,13 +19,13 @@ export const CloudHelpDropdown: React.FC = () => { { as: "a", href: links.supportPortal, - icon: , + icon: , displayName: formatMessage({ id: "sidebar.supportPortal" }), }, { as: "button", value: "inApp", - icon: , + icon: , displayName: formatMessage({ id: "sidebar.inAppHelpCenter" }), }, { @@ -39,26 +34,26 @@ export const CloudHelpDropdown: React.FC = () => { { as: "a", href: links.docsLink, - icon: , + icon: , displayName: formatMessage({ id: "sidebar.documentation" }), }, { as: "a", href: links.statusLink, - icon: , + icon: , displayName: formatMessage({ id: "sidebar.status" }), }, { as: "a", internal: true, href: CloudRoutes.UpcomingFeatures, - icon: , + icon: , displayName: formatMessage({ id: "sidebar.upcomingFeatures" }), }, ]} onChange={handleChatUs} label={} - icon={} + icon={} /> ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx index d302ee3154e..4a453bc6748 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx @@ -4,9 +4,9 @@ import { FormattedMessage } from "react-intl"; import { Outlet } from "react-router-dom"; import { LoadingPage } from "components"; -import { CreditsIcon } from "components/icons/CreditsIcon"; import { AdminWorkspaceWarning } from "components/ui/AdminWorkspaceWarning"; import { FlexItem } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { ThemeToggle } from "components/ui/ThemeToggle"; import { WorkspacesPicker } from "components/workspace/WorkspacesPicker"; @@ -27,7 +27,6 @@ import { StartOverErrorView } from "views/common/StartOverErrorView"; import { AirbyteHomeLink } from "views/layout/SideBar/AirbyteHomeLink"; import { MenuContent } from "views/layout/SideBar/components/MenuContent"; import { NavItem } from "views/layout/SideBar/components/NavItem"; -import SettingsIcon from "views/layout/SideBar/components/SettingsIcon"; import { MainNavItems } from "views/layout/SideBar/MainNavItems"; import { SideBar } from "views/layout/SideBar/SideBar"; @@ -70,7 +69,7 @@ const CloudMainView: React.FC> = (props) => { } + icon={} label={} testId="creditsButton" withNotification={ @@ -82,7 +81,7 @@ const CloudMainView: React.FC> = (props) => { } - icon={} + icon={} to={RoutePaths.Settings} /> diff --git a/airbyte-webapp/src/packages/cloud/views/settings/CloudSettingsPage.tsx b/airbyte-webapp/src/packages/cloud/views/settings/CloudSettingsPage.tsx index f688364ac7e..94b1529ea56 100644 --- a/airbyte-webapp/src/packages/cloud/views/settings/CloudSettingsPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/settings/CloudSettingsPage.tsx @@ -105,7 +105,8 @@ const CloudSettingsPage: React.FC = () => { }, ], }, - ...(canViewOrgSettings + // TODO: Org check can be removed once all workspaces are in an organization + ...(canViewOrgSettings && organization ? [ { category: , @@ -146,7 +147,7 @@ const CloudSettingsPage: React.FC = () => { : []), ], }), - [canViewOrgSettings, isSsoEnabled, supportsCloudDbtIntegration, supportsDataResidency] + [canViewOrgSettings, isSsoEnabled, organization, supportsCloudDbtIntegration, supportsDataResidency] ); return ; diff --git a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx index a48d80ce2e2..ba65d36f16d 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx @@ -9,8 +9,7 @@ import { FlexContainer } from "components/ui/Flex"; import { ModalBody, ModalFooter } from "components/ui/Modal"; import { useUserHook } from "core/api/cloud"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { trackError } from "core/utils/datadog"; import { useModalService } from "hooks/services/Modal"; import { useNotificationService } from "hooks/services/Notification"; diff --git a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx index 4650608daa7..fbed9361519 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx @@ -1,11 +1,10 @@ -import { faPlus } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { createColumnHelper } from "@tanstack/react-table"; import React, { useMemo } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { Button } from "components/ui/Button"; import { Heading } from "components/ui/Heading"; +import { Icon } from "components/ui/Icon"; import { Table } from "components/ui/Table"; import { useListUsers, useUserHook } from "core/api/cloud"; @@ -60,11 +59,7 @@ const Header: React.VFC = () => { -
diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesPage.module.scss b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesPage.module.scss index 889e2b8c9df..bb15692d616 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesPage.module.scss +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesPage.module.scss @@ -18,8 +18,3 @@ margin-right: auto; text-align: center; } - -.cloudWorkspacesPage__illustration { - display: block; - margin: auto; -} diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesPage.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesPage.tsx index 3284d6bd994..7d1a20f2ac3 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesPage.tsx @@ -8,13 +8,12 @@ import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; -import { ExternalLink } from "components/ui/Link"; import { LoadingSpinner } from "components/ui/LoadingSpinner"; import { SearchInput } from "components/ui/SearchInput"; import { Text } from "components/ui/Text"; +import { NoWorkspacePermissionsContent } from "area/workspace/NoWorkspacesPermissionWarning"; import { useListCloudWorkspacesInfinite } from "core/api/cloud"; -import { OrganizationRead } from "core/request/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useAuthService } from "core/services/auth"; import { useOrganizationsToCreateWorkspaces } from "pages/workspaces/components/useOrganizationsToCreateWorkspaces"; @@ -23,10 +22,9 @@ import { WORKSPACE_LIST_LENGTH } from "pages/workspaces/WorkspacesPage"; import { CloudWorkspacesCreateControl } from "./CloudWorkspacesCreateControl"; import styles from "./CloudWorkspacesPage.module.scss"; -import OctaviaThinking from "./octavia-thinking-no-gears.svg?react"; export const CloudWorkspacesPage: React.FC = () => { - const { isLoading, mutateAsync: handleLogout } = useMutation(() => logout?.() ?? Promise.resolve()); + const { isLoading: isLogoutLoading, mutateAsync: handleLogout } = useMutation(() => logout?.() ?? Promise.resolve()); useTrackPage(PageTrackingCodes.WORKSPACES); const [searchValue, setSearchValue] = useState(""); const [debouncedSearchValue, setDebouncedSearchValue] = useState(""); @@ -38,6 +36,7 @@ export const CloudWorkspacesPage: React.FC = () => { fetchNextPage, isFetchingNextPage, isFetching, + isLoading, } = useListCloudWorkspacesInfinite(WORKSPACE_LIST_LENGTH, debouncedSearchValue); const { organizationsMemberOnly, organizationsToCreateIn } = useOrganizationsToCreateWorkspaces(); @@ -61,13 +60,14 @@ export const CloudWorkspacesPage: React.FC = () => { 250, [searchValue] ); + console.log({ showNoWorkspacesContent }); return (
{logout && ( - )} @@ -91,43 +91,20 @@ export const CloudWorkspacesPage: React.FC = () => { - - {isFetchingNextPage && ( + + {isFetchingNextPage ? ( - )} + ) : null} )}
); }; - -const NoWorkspacePermissionsContent: React.FC<{ organizations: OrganizationRead[] }> = ({ organizations }) => { - return ( - - - -
- - - - - - - ( - {lnk} - ), - }} - /> - -
- - - ); -}; diff --git a/airbyte-webapp/src/pages/DefaultView.tsx b/airbyte-webapp/src/pages/DefaultView.tsx new file mode 100644 index 00000000000..cd2b28d1dfe --- /dev/null +++ b/airbyte-webapp/src/pages/DefaultView.tsx @@ -0,0 +1,27 @@ +import { Navigate } from "react-router-dom"; + +import { useListWorkspacesInfinite } from "core/api"; +import { FeatureItem, useFeature } from "core/services/features"; +import { RoutePaths } from "pages/routePaths"; + +export const DefaultView: React.FC = () => { + const { data: workspacesData } = useListWorkspacesInfinite(2, "", true); + const workspaces = workspacesData?.pages.flatMap((page) => page.data.workspaces) ?? []; + const multiWorkspaceUI = useFeature(FeatureItem.MultiWorkspaceUI); + + // Only show the workspace list if there is not exactly one workspace + // otherwise redirect to the single workspace + + return ( + + ); +}; + +export default DefaultView; diff --git a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx index e52256ae0c0..33fc711339a 100644 --- a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx @@ -8,8 +8,7 @@ import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; import { Box } from "components/ui/Box"; import { Card } from "components/ui/Card"; -import { useCurrentWorkspace, useUpdateOrganization } from "core/api"; -import { useOrganization } from "core/api"; +import { useCurrentWorkspace, useUpdateOrganization, useOrganization } from "core/api"; import { OrganizationUpdateRequestBody } from "core/request/AirbyteClient"; import { useIntent } from "core/utils/rbac"; import { useNotificationService } from "hooks/services/Notification"; diff --git a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx index aac824503d2..8e1ea5eb617 100644 --- a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx @@ -21,7 +21,7 @@ export const SettingsPage: React.FC = () => { const { organizationId, workspaceId } = useCurrentWorkspace(); const { countNewSourceVersion, countNewDestinationVersion } = useGetConnectorsOutOfDate(); const multiWorkspaceUI = useFeature(FeatureItem.MultiWorkspaceUI); - const isAccessManagementEnabled = false; + const isAccessManagementEnabled = useFeature(FeatureItem.RBAC); const canViewWorkspaceSettings = useIntent("ViewWorkspaceSettings", { workspaceId }); const canViewOrganizationSettings = useIntent("ViewOrganizationSettings", { organizationId }); @@ -114,7 +114,7 @@ export const SettingsPage: React.FC = () => { }, ] : []), - ...(multiWorkspaceUI + ...(multiWorkspaceUI && canViewWorkspaceSettings ? [ { category: , diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx index b6886ee206a..9bd25726a1e 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx @@ -1,10 +1,14 @@ -import { FormattedMessage } from "react-intl"; +import { FormattedMessage, useIntl } from "react-intl"; import { HeadTitle } from "components/common/HeadTitle"; import { Box } from "components/ui/Box"; import { FlexContainer } from "components/ui/Flex"; +import { Message } from "components/ui/Message"; import { Text } from "components/ui/Text"; +import { useCurrentOrganizationInfo, useCurrentWorkspace } from "core/api"; +import { useIntent } from "core/utils/rbac"; + import { AccessManagementCard } from "./AccessManagementCard"; import styles from "./AccessManagementPageContent.module.scss"; import { AccessUsers, ResourceType } from "./useGetAccessManagementData"; @@ -19,6 +23,11 @@ export const AccessManagementPageContent: React.FC accessUsers, pageResourceType, }) => { + const { formatMessage } = useIntl(); + const workspace = useCurrentWorkspace(); + const canListOrganizationUsers = useIntent("ListOrganizationMembers", { organizationId: workspace.organizationId }); + const organizationInfo = useCurrentOrganizationInfo(); + return ( <> @@ -34,6 +43,17 @@ export const AccessManagementPageContent: React.FC const users = data?.users ?? []; const usersToAdd = data?.usersToAdd ?? []; + if (resourceType === "organization" && !canListOrganizationUsers) { + return ( + + ); + } + return ( { const workspace = useCurrentWorkspace(); + const canListOrganizationUsers = useIntent("ListOrganizationMembers", { organizationId: workspace.organizationId }); const workspaceUsers = useListUsersInWorkspace(workspace.workspaceId).users; - const organizationUsers = useListUsersInOrganization(workspace.organizationId ?? "").users; + const organizationUsers = + useListUsersInOrganization(workspace.organizationId ?? "", canListOrganizationUsers)?.users ?? []; return { workspace: { diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/AccountPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/AccountPage.tsx index 49b05bf5101..da5f901f3c1 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/AccountPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/AccountPage.tsx @@ -10,7 +10,8 @@ import { FlexContainer } from "components/ui/Flex"; import { FeatureItem, useFeature } from "core/services/features"; -import AccountForm from "./components/AccountForm"; +import { AccountForm } from "./components/AccountForm"; +import { KeycloakAccountForm } from "./components/KeycloakAccountForm"; export const AccountPage: React.FC = () => { const isKeycloakAuthenticationEnabled = useFeature(FeatureItem.KeycloakAuthentication); @@ -19,9 +20,7 @@ export const AccountPage: React.FC = () => { <> }> - - - + {isKeycloakAuthenticationEnabled ? : } {isKeycloakAuthenticationEnabled && } diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx index 2702394bb75..4555fc34060 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx @@ -18,7 +18,7 @@ interface AccountFormValues { email: string; } -const AccountForm: React.FC = () => { +export const AccountForm: React.FC = () => { const { formatMessage } = useIntl(); const { registerNotification, unregisterNotificationById } = useNotificationService(); const workspace = useCurrentWorkspace(); @@ -60,5 +60,3 @@ const AccountForm: React.FC = () => { ); }; - -export default AccountForm; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/KeycloakAccountForm.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/KeycloakAccountForm.tsx new file mode 100644 index 00000000000..099dd30e29b --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/KeycloakAccountForm.tsx @@ -0,0 +1,39 @@ +import React from "react"; +import { useIntl } from "react-intl"; +import * as yup from "yup"; + +import { Form, FormControl } from "components/forms"; + +import { useCurrentUser } from "core/services/auth"; + +const accountValidationSchema = yup.object().shape({ + email: yup.string().email("form.email.error").required("form.empty.error"), +}); + +interface KeycloakAccountFormValues { + email: string; +} + +export const KeycloakAccountForm: React.FC = () => { + const { formatMessage } = useIntl(); + const user = useCurrentUser(); + + const onSubmit = async () => { + Promise.resolve(null); + }; + + return ( + + onSubmit={onSubmit} + schema={accountValidationSchema} + defaultValues={{ email: user.email ?? "" }} + disabled + > + + label={formatMessage({ id: "form.yourEmail" })} + fieldType="input" + name="email" + /> + + ); +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx index 451f9e71832..be11036a1db 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx @@ -1,13 +1,10 @@ import React, { useCallback, useMemo, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; +import { useDestinationDefinitionList, useUpdateDestinationDefinition } from "core/api"; import { DestinationDefinitionRead } from "core/request/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useNotificationService } from "hooks/services/Notification"; -import { - useDestinationDefinitionList, - useUpdateDestinationDefinition, -} from "services/connector/DestinationDefinitionService"; import ConnectorsView from "./components/ConnectorsView"; import { useDestinationList } from "../../../../hooks/services/useDestinationHook"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx index 2168383d7da..d0bb271e9ad 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx @@ -1,12 +1,11 @@ import React, { useCallback, useMemo, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { useListBuilderProjects } from "core/api"; +import { useListBuilderProjects, useSourceDefinitionList, useUpdateSourceDefinition } from "core/api"; import { SourceDefinitionRead } from "core/request/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useNotificationService } from "hooks/services/Notification"; import { useSourceList } from "hooks/services/useSourceHook"; -import { useSourceDefinitionList, useUpdateSourceDefinition } from "services/connector/SourceDefinitionService"; import ConnectorsView, { ConnectorsViewProps } from "./components/ConnectorsView"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx index 4f5e473e2fb..1343a7c0a46 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx @@ -1,6 +1,5 @@ import { createColumnHelper } from "@tanstack/react-table"; -import { useCallback, useMemo, useState } from "react"; -import React from "react"; +import React, { useCallback, useMemo, useState } from "react"; import { FormattedMessage } from "react-intl"; import { HeadTitle } from "components/common/HeadTitle"; @@ -14,6 +13,7 @@ import { BuilderProject } from "core/api"; import { Connector, ConnectorDefinition } from "core/domain/connector"; import { DestinationDefinitionRead, SourceDefinitionRead } from "core/request/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; +import { useIntent } from "core/utils/rbac"; import { RoutePaths } from "pages/routePaths"; import { ConnectorCell } from "./ConnectorCell"; @@ -66,7 +66,8 @@ const ConnectorsView: React.FC = ({ connectorBuilderProjects, }) => { const [updatingAllConnectors, setUpdatingAllConnectors] = useState(false); - const allowUpdateConnectors = useFeature(FeatureItem.AllowUpdateConnectors); + const hasUpdateConnectorsPermissions = useIntent("UpdateConnectorVersions"); + const allowUpdateConnectors = useFeature(FeatureItem.AllowUpdateConnectors) && hasUpdateConnectorsPermissions; const allowUploadCustomImage = useFeature(FeatureItem.AllowUploadCustomImage); const showVersionUpdateColumn = useCallback( diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx index 0a04e6ec456..aa7cfa0fbbb 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx @@ -1,21 +1,17 @@ -import { faDocker } from "@fortawesome/free-brands-svg-icons"; -import { faPlus } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useNavigate } from "react-router-dom"; import { Button } from "components/ui/Button"; import { DropdownMenu, DropdownMenuOptionType } from "components/ui/DropdownMenu"; +import { Icon } from "components/ui/Icon"; import { useCurrentWorkspaceId } from "area/workspace/utils"; +import { useCreateDestinationDefinition, useCreateSourceDefinition } from "core/api"; import { FeatureItem, useFeature } from "core/services/features"; import { ConnectorBuilderRoutePaths } from "pages/connectorBuilder/ConnectorBuilderRoutes"; import { DestinationPaths, RoutePaths, SourcePaths } from "pages/routePaths"; -import { useCreateDestinationDefinition } from "services/connector/DestinationDefinitionService"; -import { useCreateSourceDefinition } from "services/connector/SourceDefinitionService"; -import BuilderIcon from "./builder-icon.svg?react"; import CreateConnectorModal from "./CreateConnectorModal"; interface IProps { @@ -78,7 +74,7 @@ const CreateConnector: React.FC = ({ type }) => { { as: "a", href: `../../${RoutePaths.ConnectorBuilder}/${ConnectorBuilderRoutePaths.Create}`, - icon: , + icon: , displayName: formatMessage({ id: "admin.newConnector.build" }), internal: true, }, @@ -86,7 +82,7 @@ const CreateConnector: React.FC = ({ type }) => { ? [ { as: "button" as const, - icon: , + icon: , value: "docker", displayName: formatMessage({ id: "admin.newConnector.docker" }), }, @@ -110,7 +106,7 @@ interface NewConnectorButtonProps { const NewConnectorButton = React.forwardRef(({ onClick }, ref) => { return ( - ); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/DestinationUpdateIndicator.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/DestinationUpdateIndicator.tsx index 46baaba1227..f85ed6a93b0 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/DestinationUpdateIndicator.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/DestinationUpdateIndicator.tsx @@ -2,7 +2,7 @@ import { useMemo } from "react"; import Indicator from "components/Indicator"; -import { useLatestDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; +import { useLatestDestinationDefinitionList } from "core/api"; interface DestinationUpdateIndicatorProps { id: string; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/SourceUpdateIndicator.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/SourceUpdateIndicator.tsx index 5af5ba95eb2..68820995df6 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/SourceUpdateIndicator.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/SourceUpdateIndicator.tsx @@ -2,7 +2,7 @@ import { useMemo } from "react"; import Indicator from "components/Indicator"; -import { useLatestSourceDefinitionList } from "services/connector/SourceDefinitionService"; +import { useLatestSourceDefinitionList } from "core/api"; import { ConnectorCellProps } from "./ConnectorCell"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpdateDestinationConnectorVersionCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpdateDestinationConnectorVersionCell.tsx index d007f8ee02b..0b6b918004b 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpdateDestinationConnectorVersionCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpdateDestinationConnectorVersionCell.tsx @@ -1,4 +1,4 @@ -import { useLatestDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; +import { useLatestDestinationDefinitionList } from "core/api"; import { VersionCell, VersionCellProps } from "./VersionCell"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpdateSourceConnectorVersionCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpdateSourceConnectorVersionCell.tsx index 9f246c5c40a..dfa19c7e271 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpdateSourceConnectorVersionCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpdateSourceConnectorVersionCell.tsx @@ -1,4 +1,4 @@ -import { useLatestSourceDefinitionList } from "services/connector/SourceDefinitionService"; +import { useLatestSourceDefinitionList } from "core/api"; import { VersionCell, VersionCellProps } from "./VersionCell"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx index 7d442a8c718..d040508b6d4 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx @@ -1,9 +1,8 @@ -import { faRedoAlt } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; +import { Icon } from "components/ui/Icon"; import { useGetConnectorsOutOfDate, useUpdateAllConnectors } from "hooks/services/useConnector"; @@ -36,7 +35,7 @@ const UpgradeAllButton: React.FC = ({ connectorType }) => onClick={handleUpdateAllConnectors} isLoading={isLoading} disabled={!hasNewVersion} - icon={} + icon={} > diff --git a/airbyte-webapp/src/pages/SpeakeasyRedirectPage/SpeakeasyRedirectPage.tsx b/airbyte-webapp/src/pages/SpeakeasyRedirectPage/SpeakeasyRedirectPage.tsx index ebcf4917025..87e609d364e 100644 --- a/airbyte-webapp/src/pages/SpeakeasyRedirectPage/SpeakeasyRedirectPage.tsx +++ b/airbyte-webapp/src/pages/SpeakeasyRedirectPage/SpeakeasyRedirectPage.tsx @@ -1,5 +1,4 @@ -import { Suspense, useEffect } from "react"; -import React from "react"; +import React, { Suspense, useEffect } from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; diff --git a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.module.scss b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.module.scss index 8bced990e78..45f52c9df37 100644 --- a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.module.scss +++ b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.module.scss @@ -1,10 +1,29 @@ @use "scss/colors"; +@use "scss/variables"; -.link { - color: colors.$dark-blue; +.alignSelfStart { + align-self: flex-start; +} + +.lowercase { + text-transform: lowercase; +} + +.filterButton { + background-color: colors.$foreground; + border-radius: variables.$border-radius-sm; + color: colors.$grey-400; + border: variables.$border-thin solid colors.$grey-300; + min-height: auto; + height: variables.$button-height-xs; +} + +.filterOptionsMenu { + display: block; // default is `flex` which shrinks the options to fit the box, instead overflowing into scroll + overflow: auto; + max-height: variables.$height-long-listbox-options-list; +} - &:hover, - &:focus { - color: colors.$blue-400; - } +.filterOption { + white-space: nowrap; } diff --git a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx index 36ffc898364..72c10fc74c0 100644 --- a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx @@ -1,28 +1,180 @@ -import { faPlus } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import React, { Suspense } from "react"; +import React, { Suspense, useMemo, useState } from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; import { LoadingPage, MainPageWithScroll } from "components"; +import { ConnectorIcon } from "components/common/ConnectorIcon"; import { HeadTitle } from "components/common/HeadTitle"; import { ConnectionOnboarding } from "components/connection/ConnectionOnboarding"; +import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; +import { Card } from "components/ui/Card"; +import { FlexContainer, FlexItem } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; +import { Icon } from "components/ui/Icon"; +import { ListBox } from "components/ui/ListBox"; import { PageHeader } from "components/ui/PageHeader"; +import { Text } from "components/ui/Text"; import { useConnectionList } from "core/api"; +import { WebBackendConnectionListItem } from "core/request/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; +import { naturalComparatorBy } from "core/utils/objects"; +import { useExperiment } from "hooks/services/Experiment"; +import styles from "./AllConnectionsPage.module.scss"; import ConnectionsTable from "./ConnectionsTable"; import { ConnectionRoutePaths } from "../../routePaths"; +type SummaryKey = "healthy" | "failed" | "paused"; +const connectionStatColors: Record["color"]> = { + healthy: "green600", + failed: "red", + paused: "grey", +}; +const ConnectionsSummary: React.FC> = (props) => { + const keys = Object.keys(props) as SummaryKey[]; + const parts: React.ReactNode[] = []; + const connectionsCount = keys.reduce((total, value) => total + props[value], 0); + let consumedConnections = 0; + + for (const key of keys) { + const value = props[key]; + if (value) { + consumedConnections += value; + parts.push( + + {value} + , + consumedConnections < connectionsCount && ( + +  ·  + + ) + ); + } + } + + return <>{parts}; +}; + +interface FilterOption { + label: React.ReactNode; + value: string | null; +} + +type SortableFilterOption = FilterOption & { sortValue: string }; + +const statusFilterOptions: FilterOption[] = [ + { + label: ( + + + + ), + value: null, + }, + { + label: ( + + +   + + ), + value: "healthy", + }, + { + label: ( + + +   + + ), + value: "failed", + }, + { + label: ( + + +   + + ), + value: "paused", + }, +]; + +const isConnectionPaused = ( + connection: WebBackendConnectionListItem +): connection is WebBackendConnectionListItem & { status: "inactive" } => connection.status === "inactive"; +const isConnectionFailed = ( + connection: WebBackendConnectionListItem +): connection is WebBackendConnectionListItem & { latestSyncJobStatus: "failed" } => + connection.latestSyncJobStatus === "failed"; + export const AllConnectionsPage: React.FC = () => { const navigate = useNavigate(); useTrackPage(PageTrackingCodes.CONNECTIONS_LIST); + const isConnectionsSummaryEnabled = useExperiment("connections.summaryView", false); + const connectionList = useConnectionList(); - const connections = connectionList?.connections ?? []; + const connections = useMemo(() => connectionList?.connections ?? [], [connectionList?.connections]); + + const availableSourceOptions = getAvailableSourceOptions(connections); + const availableDestinationOptions = getAvailableDestinationOptions(connections); + + const [statusFilterSelection, setStatusFilterSelection] = useState(statusFilterOptions[0]); + const [sourceFilterSelection, setSourceFilterSelection] = useState(availableSourceOptions[0]); + const [destinationFilterSelection, setDestinationFilterSelection] = useState( + availableDestinationOptions[0] + ); + const hasAnyFilterSelected = [statusFilterSelection, sourceFilterSelection, destinationFilterSelection].some( + (selection) => !!selection.value + ); + + const filteredConnections = useMemo(() => { + const statusFilter = statusFilterSelection?.value; + const sourceFilter = sourceFilterSelection?.value; + const destinationFilter = destinationFilterSelection?.value; + + return connections.filter((connection) => { + if (statusFilter) { + const isPaused = isConnectionPaused(connection); + const isFailed = isConnectionFailed(connection); + if (statusFilter === "paused" && !isPaused) { + return false; + } else if (statusFilter === "failed" && !isFailed) { + return false; + } else if (statusFilter === "healthy" && (isPaused || isFailed)) { + return false; + } + } + + if (sourceFilter && sourceFilter !== connection.source.sourceDefinitionId) { + return false; + } + + if (destinationFilter && destinationFilter !== connection.destination.destinationDefinitionId) { + return false; + } + + return true; + }); + }, [connections, statusFilterSelection, sourceFilterSelection, destinationFilterSelection]); + + const connectionsSummary = connections.reduce>( + (acc, connection) => { + const status = isConnectionPaused(connection) ? "paused" : isConnectionFailed(connection) ? "failed" : "healthy"; + acc[status] += 1; + return acc; + }, + { + // order here governs render order + healthy: 0, + failed: 0, + paused: 0, + } + ); const onCreateClick = (sourceDefinitionId?: string) => navigate(`${ConnectionRoutePaths.ConnectionNew}`, { state: { sourceDefinitionId } }); @@ -37,25 +189,106 @@ export const AllConnectionsPage: React.FC = () => { pageTitle={ - - + + + + + + + {isConnectionsSummaryEnabled && ( + + + + )} + } endComponent={ - + + + } /> } > - + + {isConnectionsSummaryEnabled && ( + + + + + setStatusFilterSelection(statusFilterOptions.find((option) => option.value === value)!) + } + /> + + + + setSourceFilterSelection(availableSourceOptions.find((option) => option.value === value)!) + } + /> + + + + setDestinationFilterSelection( + availableDestinationOptions.find((option) => option.value === value)! + ) + } + /> + + {hasAnyFilterSelected && ( + + + + )} + + + )} + + {filteredConnections.length === 0 && ( + + + + + + )} + ) : ( @@ -64,3 +297,93 @@ export const AllConnectionsPage: React.FC = () => { ); }; + +function getAvailableSourceOptions(connections: WebBackendConnectionListItem[]) { + return connections + .reduce<{ + foundSourceIds: Set; + options: SortableFilterOption[]; + }>( + (acc, connection) => { + const { sourceName, sourceDefinitionId, icon } = connection.source; + if (acc.foundSourceIds.has(sourceDefinitionId) === false) { + acc.foundSourceIds.add(sourceDefinitionId); + acc.options.push({ + label: ( + + + + + + {sourceName} + + + ), + value: sourceDefinitionId, + sortValue: sourceName, + }); + } + return acc; + }, + { + foundSourceIds: new Set(), + options: [ + { + label: ( + + + + ), + value: null, + sortValue: "", + }, + ], + } + ) + .options.sort(naturalComparatorBy((option) => option.sortValue)); +} + +function getAvailableDestinationOptions(connections: WebBackendConnectionListItem[]) { + return connections + .reduce<{ + foundDestinationIds: Set; + options: SortableFilterOption[]; + }>( + (acc, connection) => { + const { destinationName, destinationDefinitionId, icon } = connection.destination; + if (acc.foundDestinationIds.has(destinationDefinitionId) === false) { + acc.foundDestinationIds.add(connection.destination.destinationDefinitionId); + acc.options.push({ + label: ( + + + + + + {destinationName} + + + ), + value: destinationDefinitionId, + sortValue: destinationName, + }); + } + return acc; + }, + { + foundDestinationIds: new Set(), + options: [ + { + label: ( + + + + ), + value: null, + sortValue: "", + }, + ], + } + ) + .options.sort(naturalComparatorBy((option) => option.sortValue)); +} diff --git a/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsTable.tsx b/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsTable.tsx index 2542300f202..269e7190c83 100644 --- a/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsTable.tsx +++ b/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsTable.tsx @@ -9,16 +9,17 @@ import { WebBackendConnectionListItem } from "core/request/AirbyteClient"; interface ConnectionsTableProps { connections: WebBackendConnectionListItem[]; + variant?: React.ComponentProps["variant"]; } -const ConnectionsTable: React.FC = ({ connections }) => { +const ConnectionsTable: React.FC = ({ connections, variant }) => { const navigate = useNavigate(); const data = getConnectionTableData(connections, "connection"); const clickRow = (source: ConnectionTableDataItem) => navigate(`${source.connectionId}`); - return ; + return ; }; export default ConnectionsTable; diff --git a/airbyte-webapp/src/pages/connections/ConfigureConnectionPage/ConfigureConnectionPage.tsx b/airbyte-webapp/src/pages/connections/ConfigureConnectionPage/ConfigureConnectionPage.tsx index 3e37971f4d7..0f071b91cd0 100644 --- a/airbyte-webapp/src/pages/connections/ConfigureConnectionPage/ConfigureConnectionPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConfigureConnectionPage/ConfigureConnectionPage.tsx @@ -5,11 +5,9 @@ import { Navigate, useParams, useSearchParams } from "react-router-dom"; import { LoadingPage } from "components"; import { HeadTitle } from "components/common/HeadTitle"; import { MainPageWithScroll } from "components/common/MainPageWithScroll/MainPageWithScroll"; -import { CreateConnectionForm } from "components/connection/CreateConnectionForm"; import { CreateConnectionHookForm } from "components/connection/CreateConnectionHookForm/CreateConnectionHookForm"; import { PageHeaderWithNavigation } from "components/ui/PageHeader"; -import { useExperiment } from "hooks/services/Experiment"; import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; import { CreateConnectionTitleBlock } from "../CreateConnectionPage/CreateConnectionTitleBlock"; @@ -18,7 +16,6 @@ export const ConfigureConnectionPage = () => { const { formatMessage } = useIntl(); const { workspaceId } = useParams<{ workspaceId: string }>(); const [searchParams] = useSearchParams(); - const doUseCreateConnectionHookForm = useExperiment("form.createConnectionHookForm", false); const sourceId = searchParams.get("sourceId"); const destinationId = searchParams.get("destinationId"); @@ -52,7 +49,7 @@ export const ConfigureConnectionPage = () => { } > }> - {doUseCreateConnectionHookForm ? : } + ); diff --git a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.module.scss b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.module.scss index 03d55ed98f6..a0fe1dd485c 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.module.scss +++ b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.module.scss @@ -20,9 +20,3 @@ align-items: center; justify-content: center; } - -// To be removed after connection.searchableJobLogs experiment is merged -.narrowTable { - max-width: variables.$page-width; - margin-inline: auto; -} diff --git a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.tsx index 8d5e127cc88..d1a97338d3a 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.tsx @@ -1,4 +1,3 @@ -import classNames from "classnames"; import React, { useState } from "react"; import { FormattedMessage } from "react-intl"; import { useLocation } from "react-router-dom"; @@ -6,17 +5,22 @@ import { useLocation } from "react-router-dom"; import { EmptyResourceBlock } from "components/common/EmptyResourceBlock"; import { ConnectionSyncButtons } from "components/connection/ConnectionSync/ConnectionSyncButtons"; import { ConnectionSyncContextProvider } from "components/connection/ConnectionSync/ConnectionSyncContext"; -import { useAttemptLink } from "components/JobItem/attemptLinkUtils"; +import { PageContainer } from "components/PageContainer"; import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; import { Link } from "components/ui/Link"; +import { useAttemptLink } from "area/connection/utils/attemptLink"; import { useListJobs } from "core/api"; -import { getFrequencyFromScheduleData } from "core/services/analytics"; -import { Action, Namespace } from "core/services/analytics"; -import { useTrackPage, PageTrackingCodes, useAnalyticsService } from "core/services/analytics"; +import { + getFrequencyFromScheduleData, + Action, + Namespace, + useTrackPage, + PageTrackingCodes, + useAnalyticsService, +} from "core/services/analytics"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; -import { useExperiment } from "hooks/services/Experiment"; import styles from "./ConnectionJobHistoryPage.module.scss"; import JobsList from "./JobsList"; @@ -41,7 +45,6 @@ export const ConnectionJobHistoryPage: React.FC = () => { pageSize: jobPageSize, }, }); - const searchableJobLogsEnabled = useExperiment("connection.searchableJobLogs", true); const linkedJobNotFound = linkedJobId && jobs.length === 0; const moreJobPagesAvailable = !linkedJobNotFound && jobPageSize < totalJobCount; @@ -62,7 +65,7 @@ export const ConnectionJobHistoryPage: React.FC = () => { }; return ( -
+ { )} -
+ ); }; diff --git a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/JobsList.tsx b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/JobsList.tsx index e44424deb2b..0a58ab1db21 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/JobsList.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/JobsList.tsx @@ -1,19 +1,14 @@ import React, { useMemo } from "react"; -import { JobItem } from "components/JobItem"; -import { JobWithAttempts } from "components/JobItem/types"; -import { NewJobItem } from "components/NewJobItem"; - +import { JobHistoryItem } from "area/connection/components/JobHistoryItem"; +import { JobWithAttempts } from "area/connection/types/jobs"; import { JobWithAttemptsRead } from "core/request/AirbyteClient"; -import { useExperiment } from "hooks/services/Experiment"; interface JobsListProps { jobs: JobWithAttemptsRead[]; } const JobsList: React.FC = ({ jobs }) => { - const searchableJobLogsEnabled = useExperiment("connection.searchableJobLogs", true); - const sortJobReads: JobWithAttempts[] = useMemo( () => jobs @@ -24,12 +19,9 @@ const JobsList: React.FC = ({ jobs }) => { return (
- {searchableJobLogsEnabled && - sortJobReads.map((jobWithAttempts) => ( - - ))} - {!searchableJobLogsEnabled && - sortJobReads.map((jobWithAttempts) => )} + {sortJobReads.map((jobWithAttempts) => ( + + ))}
); }; diff --git a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPage.tsx index 3d219fa74e3..ae771fe5d93 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPage.tsx @@ -6,12 +6,11 @@ import { HeadTitle } from "components/common/HeadTitle"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; -import { ConnectionEditHookFormServiceProvider } from "hooks/services/ConnectionEdit/ConnectionEditHookFormService"; import { ConnectionEditServiceProvider, useConnectionEditService, } from "hooks/services/ConnectionEdit/ConnectionEditService"; -import { useExperiment, useExperimentContext } from "hooks/services/Experiment"; +import { useExperimentContext } from "hooks/services/Experiment"; import { ConnectionRoutePaths } from "pages/routePaths"; import { ResourceNotFoundErrorBoundary } from "views/common/ResourceNotFoundErrorBoundary"; import { StartOverErrorView } from "views/common/StartOverErrorView"; @@ -52,13 +51,8 @@ export const ConnectionPage: React.FC = () => { useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM); - const doUseCreateConnectionHookForm = useExperiment("form.createConnectionHookForm", false); - const ConnectionEditServiceContextProvider = doUseCreateConnectionHookForm - ? ConnectionEditHookFormServiceProvider - : ConnectionEditServiceProvider; - return ( - + } trackError={trackError}> } @@ -70,6 +64,6 @@ export const ConnectionPage: React.FC = () => { - + ); }; diff --git a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageHeader.tsx b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageHeader.tsx index 372eff44eea..9bf4ccd940c 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageHeader.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageHeader.tsx @@ -8,8 +8,7 @@ import { PageHeaderWithNavigation } from "components/ui/PageHeader"; import { Tabs, LinkTab } from "components/ui/Tabs"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; -import { RoutePaths } from "pages/routePaths"; -import { ConnectionRoutePaths } from "pages/routePaths"; +import { RoutePaths, ConnectionRoutePaths } from "pages/routePaths"; import { ConnectionTitleBlock } from "./ConnectionTitleBlock"; diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationHookFormPage.test.tsx similarity index 90% rename from airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx rename to airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationHookFormPage.test.tsx index 2377dc580ac..27fa5544e4e 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationHookFormPage.test.tsx @@ -1,10 +1,6 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/no-non-null-assertion */ - import { render as tlr, act } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React, { Suspense } from "react"; -import selectEvent from "react-select-event"; import { VirtuosoMockContext } from "react-virtuoso"; import { mockConnection } from "test-utils/mock-data/mockConnection"; @@ -28,16 +24,9 @@ import { WebBackendConnectionUpdate } from "core/api/types/AirbyteClient"; import { defaultOssFeatures, FeatureItem } from "core/services/features"; import { ConnectionEditServiceProvider } from "hooks/services/ConnectionEdit/ConnectionEditService"; +import { ConnectionReplicationHookFormPage } from "./ConnectionReplicationHookFormPage"; import { ConnectionReplicationPage } from "./ConnectionReplicationPage"; -jest.mock("services/connector/SourceDefinitionService", () => ({ - useSourceDefinition: () => mockSourceDefinition, -})); - -jest.mock("services/connector/DestinationDefinitionService", () => ({ - useDestinationDefinition: () => mockDestinationDefinition, -})); - jest.setTimeout(40000); jest.mock("area/workspace/utils", () => ({ @@ -57,13 +46,15 @@ jest.mock("core/api", () => ({ useDestinationDefinitionVersion: () => mockDestinationDefinitionVersion, useGetSourceDefinitionSpecification: () => mockSourceDefinitionSpecification, useGetDestinationDefinitionSpecification: () => mockDestinationDefinitionSpecification, + useSourceDefinition: () => mockSourceDefinition, + useDestinationDefinition: () => mockDestinationDefinition, })); jest.mock("hooks/theme/useAirbyteTheme", () => ({ useAirbyteTheme: () => mockTheme, })); -describe("ConnectionReplicationPage", () => { +describe("ConnectionReplicationHookFormPage", () => { const Wrapper: React.FC> = ({ children }) => ( I should not show up in a snapshot
}> @@ -82,7 +73,7 @@ describe("ConnectionReplicationPage", () => { await act(async () => { renderResult = tlr( - + ); }); @@ -139,7 +130,8 @@ describe("ConnectionReplicationPage", () => { await userEvent.click(renderResult.getByTestId("configuration-card-expand-arrow")); - await selectEvent.select(renderResult.getByTestId("scheduleData"), /cron/i); + await userEvent.click(renderResult.getByTestId("schedule-type-listbox-button")); + await userEvent.click(renderResult.getByTestId("cron-option")); const cronExpressionInput = renderResult.getByTestId("cronExpression"); @@ -158,7 +150,8 @@ describe("ConnectionReplicationPage", () => { await userEvent.click(renderResult.getByTestId("configuration-card-expand-arrow")); - await selectEvent.select(renderResult.getByTestId("scheduleData"), /cron/i); + await userEvent.click(renderResult.getByTestId("schedule-type-listbox-button")); + await userEvent.click(renderResult.getByTestId("cron-option")); const cronExpressionField = renderResult.getByTestId("cronExpression"); @@ -185,7 +178,8 @@ describe("ConnectionReplicationPage", () => { await userEvent.click(container.getByTestId("configuration-card-expand-arrow")); - await selectEvent.select(container.getByTestId("scheduleData"), /cron/i); + await userEvent.click(container.getByTestId("schedule-type-listbox-button")); + await userEvent.click(container.getByTestId("cron-option")); const cronExpressionField = container.getByTestId("cronExpression"); diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationHookFormPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationHookFormPage.tsx index 837b3098acc..1b43f1be387 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationHookFormPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationHookFormPage.tsx @@ -22,19 +22,24 @@ import { Message } from "components/ui/Message/Message"; import { ConnectionValues, useGetStateTypeQuery } from "core/api"; import { + AirbyteStreamAndConfiguration, AirbyteStreamConfiguration, WebBackendConnectionRead, WebBackendConnectionUpdate, } from "core/api/types/AirbyteClient"; -import { getFrequencyFromScheduleData } from "core/services/analytics"; -import { Action, Namespace } from "core/services/analytics"; -import { PageTrackingCodes, useAnalyticsService, useTrackPage } from "core/services/analytics"; +import { + getFrequencyFromScheduleData, + Action, + Namespace, + PageTrackingCodes, + useAnalyticsService, + useTrackPage, +} from "core/services/analytics"; import { equal } from "core/utils/objects"; import { useConfirmCatalogDiff } from "hooks/connection/useConfirmCatalogDiff"; import { useSchemaChanges } from "hooks/connection/useSchemaChanges"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; import { useModalService } from "hooks/services/Modal"; import styles from "./ConnectionReplicationPage.module.scss"; @@ -103,7 +108,7 @@ export const ConnectionReplicationHookFormPage: React.FC = () => { const { openModal } = useModalService(); const { connection, schemaRefreshing, updateConnection, discardRefreshedSchema } = useConnectionEditService(); - const { initialValues, schemaError, setSubmitError, refreshSchema } = useConnectionHookFormService(); + const { initialValues, schemaError, setSubmitError, refreshSchema } = useConnectionFormService(); const validationSchema = useConnectionHookFormValidationSchema(); const saveConnection = useCallback( @@ -157,18 +162,17 @@ export const ConnectionReplicationHookFormPage: React.FC = () => { // for each form value stream, find the corresponding connection value stream // and remove `config.selected` from both before comparing // we need to reset if any of the streams are different - type SyncSchemaStream = (typeof values.syncCatalog.streams)[number]; - - const getStreamId = (stream: SyncSchemaStream) => { + const getStreamId = (stream: AirbyteStreamAndConfiguration) => { return `${stream.stream?.namespace ?? ""}-${stream.stream?.name}`; }; - const lookupConnectionValuesStreamById = connection.syncCatalog.streams.reduce>( - (agg, stream) => { - agg[getStreamId(stream)] = stream; - return agg; - }, - {} - ); + + const lookupConnectionValuesStreamById = connection.syncCatalog.streams.reduce< + Record + >((agg, stream) => { + agg[getStreamId(stream)] = stream; + return agg; + }, {}); + const hasUserChangesInEnabledStreamsRequiringReset = values.syncCatalog.streams.some((_stream) => { const formStream = structuredClone(_stream); const connectionStream = structuredClone(lookupConnectionValuesStreamById[getStreamId(formStream)]); diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx index 60b007454ee..0e96375528a 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx @@ -1,319 +1,7 @@ -import { Form, Formik, FormikHelpers, useFormikContext } from "formik"; -import React, { useCallback, useEffect, useState } from "react"; -import { FormattedMessage, useIntl } from "react-intl"; -import { useLocation } from "react-router-dom"; -import { useUnmount } from "react-use"; - -import { ConnectionFormFields } from "components/connection/ConnectionForm/ConnectionFormFields"; -import EditControls from "components/connection/ConnectionForm/EditControls"; -import { - FormikConnectionFormValues, - useConnectionValidationSchema, -} from "components/connection/ConnectionForm/formConfig"; -import { useRefreshSourceSchemaWithConfirmationOnDirty } from "components/connection/ConnectionForm/refreshSourceSchemaWithConfirmationOnDirty"; -import { SchemaChangeBackdrop } from "components/connection/ConnectionForm/SchemaChangeBackdrop"; -import { SchemaError } from "components/connection/CreateConnectionForm/SchemaError"; -import LoadingSchema from "components/LoadingSchema"; -import { FlexContainer } from "components/ui/Flex"; -import { Message } from "components/ui/Message/Message"; - -import { useCurrentWorkspaceId } from "area/workspace/utils"; -import { ConnectionValues, useGetStateTypeQuery } from "core/api"; -import { SchemaChange, WebBackendConnectionRead, WebBackendConnectionUpdate } from "core/api/types/AirbyteClient"; -import { getFrequencyFromScheduleData } from "core/services/analytics"; -import { Action, Namespace } from "core/services/analytics"; -import { PageTrackingCodes, useAnalyticsService, useTrackPage } from "core/services/analytics"; -import { equal } from "core/utils/objects"; -import { useConfirmCatalogDiff } from "hooks/connection/useConfirmCatalogDiff"; -import { useSchemaChanges } from "hooks/connection/useSchemaChanges"; -import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; -import { - tidyConnectionFormValues, - useConnectionFormService, -} from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useExperiment } from "hooks/services/Experiment"; -import { useModalService } from "hooks/services/Modal"; +import React from "react"; import { ConnectionReplicationHookFormPage } from "./ConnectionReplicationHookFormPage"; -import styles from "./ConnectionReplicationPage.module.scss"; -import { ResetWarningModal } from "./ResetWarningModal"; - -const toWebBackendConnectionUpdate = (connection: WebBackendConnectionRead): WebBackendConnectionUpdate => ({ - name: connection.name, - connectionId: connection.connectionId, - namespaceDefinition: connection.namespaceDefinition, - namespaceFormat: connection.namespaceFormat, - prefix: connection.prefix, - syncCatalog: connection.syncCatalog, - scheduleData: connection.scheduleData, - scheduleType: connection.scheduleType, - status: connection.status, - resourceRequirements: connection.resourceRequirements, - operations: connection.operations, - sourceCatalogId: connection.catalogId, -}); - -const ValidateFormOnSchemaRefresh: React.FC = () => { - const { schemaHasBeenRefreshed } = useConnectionEditService(); - const { setTouched } = useFormikContext(); - - useEffect(() => { - if (schemaHasBeenRefreshed) { - setTouched({ syncCatalog: true }, true); - } - }, [setTouched, schemaHasBeenRefreshed]); - - return null; -}; - -const SchemaChangeMessage: React.FC<{ dirty: boolean; schemaChange: SchemaChange }> = ({ dirty, schemaChange }) => { - const { hasNonBreakingSchemaChange, hasBreakingSchemaChange } = useSchemaChanges(schemaChange); - const { schemaHasBeenRefreshed } = useConnectionEditService(); - const { refreshSchema } = useConnectionFormService(); - const refreshWithConfirm = useRefreshSourceSchemaWithConfirmationOnDirty(dirty); - - if (schemaHasBeenRefreshed) { - return null; - } // todo: note in review that this is a behavior change - - if (hasNonBreakingSchemaChange) { - return ( - } - actionBtnText={} - onAction={refreshSchema} - data-testid="schemaChangesDetected" - /> - ); - } - - if (hasBreakingSchemaChange) { - return ( - } - actionBtnText={} - onAction={refreshWithConfirm} - data-testid="schemaChangesDetected" - /> - ); - } - return null; -}; - -const ConnectionReplicationFormikFormPage: React.FC = () => { - const analyticsService = useAnalyticsService(); - const getStateType = useGetStateTypeQuery(); - const workspaceId = useCurrentWorkspaceId(); - - const { formatMessage } = useIntl(); - const { openModal } = useModalService(); - - const [saved, setSaved] = useState(false); - - const { connection, schemaRefreshing, schemaHasBeenRefreshed, updateConnection, discardRefreshedSchema } = - useConnectionEditService(); - const { initialValues, mode, schemaError, getErrorMessage, setSubmitError, refreshSchema } = - useConnectionFormService(); - const validationSchema = useConnectionValidationSchema({ mode }); - - useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_REPLICATION, { stream_count: connection.syncCatalog.streams.length }); - - const saveConnection = useCallback( - async ( - values: ConnectionValues, - { skipReset, catalogHasChanged }: { skipReset: boolean; catalogHasChanged: boolean } - ) => { - const connectionAsUpdate = toWebBackendConnectionUpdate(connection); - - await updateConnection({ - ...connectionAsUpdate, - ...values, - connectionId: connection.connectionId, - skipReset, - }); - - if (catalogHasChanged) { - // TODO (https://github.com/airbytehq/airbyte/issues/17666): Move this into a useTrackChangedCatalog method (name pending) post Vlad's analytics hook work - analyticsService.track(Namespace.CONNECTION, Action.EDIT_SCHEMA, { - actionDescription: "Connection saved with catalog changes", - connector_source: connection.source.sourceName, - connector_source_definition_id: connection.source.sourceDefinitionId, - connector_destination: connection.destination.destinationName, - connector_destination_definition_id: connection.destination.destinationDefinitionId, - frequency: getFrequencyFromScheduleData(connection.scheduleData), - }); - } - }, - [analyticsService, connection, updateConnection] - ); - - const onFormSubmit = useCallback( - async (values: FormikConnectionFormValues, _: FormikHelpers) => { - const formValues = tidyConnectionFormValues(values, workspaceId, validationSchema, connection.operations); - - // Check if the user refreshed the catalog and there was any change in a currently enabled stream - const hasDiffInEnabledStream = connection.catalogDiff?.transforms.some(({ streamDescriptor }) => { - // Find the stream for this transform in our form's syncCatalog - const stream = formValues.syncCatalog.streams.find( - ({ stream }) => streamDescriptor.name === stream?.name && streamDescriptor.namespace === stream.namespace - ); - return stream?.config?.selected; - }); - - // Check if the user made any modifications to enabled streams compared to the ones in the latest connection - // e.g. changed the sync mode of an enabled stream - const hasUserChangesInEnabledStreams = !equal( - formValues.syncCatalog.streams.filter((s) => s.config?.selected), - connection.syncCatalog.streams.filter((s) => s.config?.selected) - ); - - // Only adding/removing a stream - with 0 other changes - doesn't require a reset - // for each form value stream, find the corresponding connection value stream - // and remove `config.selected` from both before comparing - // we need to reset if any of the streams are different - type SyncSchemaStream = (typeof formValues.syncCatalog.streams)[number]; - - const getStreamId = (stream: SyncSchemaStream) => { - return `${stream.stream?.namespace ?? ""}-${stream.stream?.name}`; - }; - const lookupConnectionValuesStreamById = connection.syncCatalog.streams.reduce>( - (agg, stream) => { - agg[getStreamId(stream)] = stream; - return agg; - }, - {} - ); - const hasUserChangesInEnabledStreamsRequiringReset = formValues.syncCatalog.streams.some((_stream) => { - const formStream = structuredClone(_stream); - const connectionStream = structuredClone(lookupConnectionValuesStreamById[getStreamId(formStream)]); - - delete formStream.config?.selected; - delete connectionStream.config?.selected; - - return !equal(formStream, connectionStream); - }); - - const catalogHasChanged = hasDiffInEnabledStream || hasUserChangesInEnabledStreams; - const catalogChangesRequireReset = hasDiffInEnabledStream || hasUserChangesInEnabledStreamsRequiringReset; - - setSubmitError(null); - - // Whenever the catalog changed show a warning to the user, that we're about to reset their data. - // Given them a choice to opt-out in which case we'll be sending skipReset: true to the update - // endpoint. - try { - if (catalogChangesRequireReset) { - const stateType = await getStateType(connection.connectionId); - const result = await openModal({ - title: formatMessage({ id: "connection.resetModalTitle" }), - size: "md", - content: (props) => , - }); - if (result.type !== "canceled") { - // Save the connection taking into account the correct skipReset value from the dialog choice. - await saveConnection(formValues, { - skipReset: !result.reason, - catalogHasChanged, - }); - } else { - // We don't want to set saved to true or schema has been refreshed to false. - return; - } - } else { - // The catalog hasn't changed, or only added/removed stream(s). We don't need to ask for any confirmation and can simply save. - await saveConnection(formValues, { skipReset: true, catalogHasChanged }); - } - - setSaved(true); - } catch (e) { - setSubmitError(e); - } - }, - [ - workspaceId, - validationSchema, - connection.operations, - connection.catalogDiff?.transforms, - connection.syncCatalog.streams, - connection.connectionId, - setSubmitError, - getStateType, - openModal, - formatMessage, - saveConnection, - ] - ); - - useConfirmCatalogDiff(); - - useUnmount(() => { - discardRefreshedSchema(); - }); - - const { state } = useLocation(); - useEffect(() => { - if (typeof state === "object" && state && "triggerRefreshSchema" in state && state.triggerRefreshSchema) { - refreshSchema(); - } - }, [refreshSchema, state]); - - return ( -
- {schemaError && !schemaRefreshing ? ( - - ) : !schemaRefreshing && connection ? ( - - {({ isSubmitting, isValid, dirty, resetForm, status, errors }) => ( - - - -
- - -
-
- - - - )} - - ) : ( - - )} -
- ); -}; export const ConnectionReplicationPage: React.FC = () => { - /** - *TODO: remove conditional component rendering after successful CreateConnectionForm migration - *https://github.com/airbytehq/airbyte-platform-internal/issues/8639 - * didn't want to add conditional logic to routes file, decided to do it here, since it's temporary - */ - const doUseCreateConnectionHookForm = useExperiment("form.createConnectionHookForm", false); - if (doUseCreateConnectionHookForm) { - return ; - } - return ; + return ; }; diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationHookFormPage.test.tsx.snap b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationHookFormPage.test.tsx.snap new file mode 100644 index 00000000000..256e608f8fe --- /dev/null +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationHookFormPage.test.tsx.snap @@ -0,0 +1,827 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ConnectionReplicationHookFormPage should render 1`] = ` + +
+
+
+
+
+
+ +
+
+
+

+ Activate the streams you want to sync +

+ +
+
+
+
+ +
+
+
+ + +
+
+
+
+ +

+ Sync +

+
+
+

+ Data destination + +

+
+
+

+ Stream + +

+
+
+

+ Sync mode + + + +

+ + + +

+
+
+

+

+
+

+ Fields +

+
+
+
+
+
+
+
+
+ +
+
+
+ +

+ '<destination schema> +

+ +
+
+ +

+ another_stream +

+ +
+
+
+ +
+
+
+
+
+
+
+

+ All +

+
+
+
+
+
+
+
+
+ +
+
+
+ +

+ '<destination schema> +

+ +
+
+ +

+ pokemon +

+ +
+
+
+ +
+
+
+
+
+
+
+

+ All +

+
+
+
+
+
+
+
+
+ +
+
+
+ +

+ '<destination schema> +

+ +
+
+ +

+ pokemon2 +

+ +
+
+
+ +
+
+
+
+
+
+
+

+ All +

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ + +
+
+
+
+
+
+ +
+
+
+ +`; + +exports[`ConnectionReplicationHookFormPage should show an error if there is a schemaError 1`] = ` + +
+
+
+
+
+ +
+
+
+
+
+
+ + Sorry. Something went wrong... + +
+
+
+
+
+
+
+ +`; diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap deleted file mode 100644 index d00a4d97aaf..00000000000 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap +++ /dev/null @@ -1,877 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`ConnectionReplicationPage should render 1`] = ` - -
-
-
-
-
-
- -
-
-
-

- Activate the streams you want to sync -

- -
-
-
-
- -
-
-
- - -
-
-
-
-
-
- -

- Sync -

-
-
-

- Data destination - -

-
-
-

- Stream - -

-
-
-

- Sync mode - - - - - - - - - -

-
-
-

-

-
-

- Fields -

-
-
-
-
-
-
-
-
-
- -
-
-
- -

- '<destination schema> -

- -
-
- -

- another_stream -

- -
-
-
- -
-
-
-
-
-
-
-

- All -

-
-
-
-
-
-
-
-
- -
-
-
- -

- '<destination schema> -

- -
-
- -

- pokemon -

- -
-
-
- -
-
-
-
-
-
-
-

- All -

-
-
-
-
-
-
-
-
- -
-
-
- -

- '<destination schema> -

- -
-
- -

- pokemon2 -

- -
-
-
- -
-
-
-
-
-
-
-

- All -

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- - -
-
-
- -
-
-
-
- -`; - -exports[`ConnectionReplicationPage should show an error if there is a schemaError 1`] = ` - -
-
-
-
-
- -
-
-
- -
-
- - Sorry. Something went wrong... - -
-
-
-
-
-
-
- -`; diff --git a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx index 1876f6b8de6..153f6b3236e 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx @@ -1,5 +1,3 @@ -import { faChevronDown, faChevronRight } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { Disclosure } from "@headlessui/react"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; @@ -13,6 +11,7 @@ import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; +import { Icon } from "components/ui/Icon"; import { ExternalLink } from "components/ui/Link"; import { Spinner } from "components/ui/Spinner"; @@ -150,7 +149,7 @@ export const ConnectionSettingsPage: React.FC = () => { } + icon={} className={styles.advancedButton} > diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsForm.tsx b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsForm.tsx index 5e87d669d0d..baa554eb54d 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsForm.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/CustomTransformationsForm.tsx @@ -3,7 +3,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; import { SchemaOf } from "yup"; -import { getInitialTransformations } from "components/connection/ConnectionForm/formConfig"; +import { getInitialTransformations } from "components/connection/ConnectionForm/hookFormConfig"; import { TransformationFieldHookForm } from "components/connection/ConnectionForm/TransformationFieldHookForm"; import { DbtOperationReadOrCreate, dbtOperationReadOrCreateSchema } from "components/connection/TransformationHookForm"; import { Form } from "components/forms"; diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/NormalizationForm.tsx b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/NormalizationForm.tsx index 724e6ad6cdd..c310820af42 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/NormalizationForm.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/NormalizationForm.tsx @@ -3,7 +3,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; import { SchemaOf } from "yup"; -import { getInitialNormalization, mapFormPropsToOperation } from "components/connection/ConnectionForm/formConfig"; +import { getInitialNormalization } from "components/connection/ConnectionForm/hookFormConfig"; import { NormalizationHookFormField } from "components/connection/ConnectionForm/NormalizationHookFormField"; import { Form } from "components/forms"; import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; @@ -12,6 +12,7 @@ import { CollapsibleCard } from "components/ui/CollapsibleCard"; import { NormalizationType } from "area/connection/types"; import { isNormalizationTransformation } from "area/connection/utils"; import { useCurrentWorkspace } from "core/api"; +import { OperatorType } from "core/api/types/AirbyteClient"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; @@ -44,14 +45,30 @@ export const NormalizationForm: React.FC = () => { [operations] ); - const onSubmit = async (values: NormalizationFormValues) => { - const normalizationOperation = mapFormPropsToOperation(values, operations, workspaceId); - + const onSubmit = async ({ normalization }: NormalizationFormValues) => { const operationsWithoutNormalization = (operations ?? [])?.filter((op) => !isNormalizationTransformation(op)); await updateConnection({ connectionId, - operations: [...normalizationOperation, ...operationsWithoutNormalization], + operations: [ + // if normalization is "basic", add normalization operation + ...(normalization === NormalizationType.basic + ? [ + { + name: "Normalization", + workspaceId, + operatorConfiguration: { + operatorType: OperatorType.normalization, + normalization: { + option: normalization, + }, + }, + }, + ] + : // if normalization is "raw", remove normalization operation + []), + ...operationsWithoutNormalization, + ], }); }; diff --git a/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx b/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx index a12cf5f3a06..e256cdac2dc 100644 --- a/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx +++ b/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx @@ -10,13 +10,17 @@ import { NumberBadge } from "components/ui/NumberBadge"; import { SupportLevelBadge } from "components/ui/SupportLevelBadge"; import { Text } from "components/ui/Text"; -import { useCurrentWorkspace, useDestinationDefinitionVersion, useSourceDefinitionVersion } from "core/api"; +import { + useCurrentWorkspace, + useDestinationDefinitionVersion, + useSourceDefinitionVersion, + useSourceDefinition, + useDestinationDefinition, +} from "core/api"; import { SupportLevel } from "core/request/AirbyteClient"; import { useGetDestination } from "hooks/services/useDestinationHook"; import { useGetSource } from "hooks/services/useSourceHook"; import { RoutePaths } from "pages/routePaths"; -import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; -import { useSourceDefinition } from "services/connector/SourceDefinitionService"; import styles from "./CreateConnectionTitleBlock.module.scss"; @@ -193,7 +197,7 @@ export const CreateConnectionTitleBlock: React.FC = () => { return ( - {idx !== Object.keys(stepStatuses).length - 1 && } + {idx !== Object.keys(stepStatuses).length - 1 && } ); })} diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusOverview.tsx b/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusOverview.tsx index 663b140c0c4..afffa4b0694 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusOverview.tsx +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusOverview.tsx @@ -38,7 +38,11 @@ export const ConnectionStatusOverview: React.FC = () => { {status === ConnectionStatusIndicatorStatus.OnTrack && ( - } placement="top"> + } + placement="top" + > = ({ streamStat return ( - {() => ), cell: (props) => ( diff --git a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderCreatePage/ConnectorBuilderCreatePage.tsx b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderCreatePage/ConnectorBuilderCreatePage.tsx index 295733a69af..c1992a94c63 100644 --- a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderCreatePage/ConnectorBuilderCreatePage.tsx +++ b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderCreatePage/ConnectorBuilderCreatePage.tsx @@ -1,5 +1,3 @@ -import { faArrowRight } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { load, YAMLException } from "js-yaml"; import lowerCase from "lodash/lowerCase"; import startCase from "lodash/startCase"; @@ -20,8 +18,7 @@ import { Text } from "components/ui/Text"; import { useListBuilderProjects } from "core/api"; import { ConnectorManifest } from "core/api/types/ConnectorManifest"; -import { Action, Namespace } from "core/services/analytics"; -import { useAnalyticsService } from "core/services/analytics"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { links } from "core/utils/links"; import { useNotificationService } from "hooks/services/Notification"; import { ConnectorBuilderLocalStorageProvider } from "services/connectorBuilder/ConnectorBuilderLocalStorageService"; @@ -180,7 +177,7 @@ const ConnectorBuilderCreatePageInner: React.FC = () => { - + @@ -241,7 +238,7 @@ const Tile: React.FC = ({ image, title, description, buttonText, butt diff --git a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderListPage/ConnectorBuilderListPage.tsx b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderListPage/ConnectorBuilderListPage.tsx index 9d011368a9b..eae7ec2eb9f 100644 --- a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderListPage/ConnectorBuilderListPage.tsx +++ b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderListPage/ConnectorBuilderListPage.tsx @@ -1,5 +1,3 @@ -import { faPlus } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { FormattedMessage } from "react-intl"; import { Navigate, useNavigate } from "react-router-dom"; @@ -8,6 +6,7 @@ import { HeadTitle } from "components/common/HeadTitle"; import { ConnectorBuilderProjectTable } from "components/ConnectorBuilderProjectTable"; import { Button } from "components/ui/Button"; import { Heading } from "components/ui/Heading"; +import { Icon } from "components/ui/Icon"; import { PageHeader } from "components/ui/PageHeader"; import { useListBuilderProjects } from "core/api"; @@ -30,7 +29,7 @@ export const ConnectorBuilderListPage: React.FC = () => { } endComponent={ } diff --git a/airbyte-webapp/src/pages/destination/CreateDestinationPage/CreateDestinationPage.tsx b/airbyte-webapp/src/pages/destination/CreateDestinationPage/CreateDestinationPage.tsx index 33fa06ad184..469dd3e4c30 100644 --- a/airbyte-webapp/src/pages/destination/CreateDestinationPage/CreateDestinationPage.tsx +++ b/airbyte-webapp/src/pages/destination/CreateDestinationPage/CreateDestinationPage.tsx @@ -13,12 +13,11 @@ import { Icon } from "components/ui/Icon"; import { PageHeaderWithNavigation } from "components/ui/PageHeader"; import { ConnectionConfiguration } from "area/connector/types"; +import { useDestinationDefinitionList } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { useCreateDestination } from "hooks/services/useDestinationHook"; -import { DestinationPaths } from "pages/routePaths"; -import { RoutePaths } from "pages/routePaths"; -import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; +import { DestinationPaths, RoutePaths } from "pages/routePaths"; import { ConnectorDocumentationWrapper } from "views/Connector/ConnectorDocumentationLayout"; export const CreateDestinationPage: React.FC = () => { diff --git a/airbyte-webapp/src/pages/destination/DestinationItemPage/DestinationItemPage.tsx b/airbyte-webapp/src/pages/destination/DestinationItemPage/DestinationItemPage.tsx index 90e7dd5bd72..b3e99f5eda5 100644 --- a/airbyte-webapp/src/pages/destination/DestinationItemPage/DestinationItemPage.tsx +++ b/airbyte-webapp/src/pages/destination/DestinationItemPage/DestinationItemPage.tsx @@ -11,11 +11,10 @@ import { StepsTypes } from "components/ConnectorBlocks"; import { PageHeaderWithNavigation } from "components/ui/PageHeader"; import { useGetDestinationFromParams } from "area/connector/utils"; -import { useDestinationDefinitionVersion } from "core/api"; +import { useDestinationDefinitionVersion, useDestinationDefinition } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { RoutePaths } from "pages/routePaths"; -import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; import { ResourceNotFoundErrorBoundary } from "views/common/ResourceNotFoundErrorBoundary"; import { StartOverErrorView } from "views/common/StartOverErrorView"; import { ConnectorDocumentationWrapper } from "views/Connector/ConnectorDocumentationLayout"; diff --git a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx index 9a441931f75..ee5b1f5b987 100644 --- a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx @@ -5,7 +5,12 @@ import { Box } from "components/ui/Box"; import { Text } from "components/ui/Text"; import { useGetDestinationFromParams } from "area/connector/utils"; -import { useConnectionList, useDestinationDefinitionVersion, useGetDestinationDefinitionSpecification } from "core/api"; +import { + useConnectionList, + useDestinationDefinitionVersion, + useGetDestinationDefinitionSpecification, + useDestinationDefinition, +} from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { @@ -14,7 +19,6 @@ import { useUpdateDestination, } from "hooks/services/useDestinationHook"; import { useDeleteModal } from "hooks/useDeleteModal"; -import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; import { ConnectorCard } from "views/Connector/ConnectorCard"; import { ConnectorCardValues } from "views/Connector/ConnectorForm/types"; diff --git a/airbyte-webapp/src/pages/destination/SelectDestinationPage/SelectDestinationPage.tsx b/airbyte-webapp/src/pages/destination/SelectDestinationPage/SelectDestinationPage.tsx index 0e0d53c64db..978be7c7afa 100644 --- a/airbyte-webapp/src/pages/destination/SelectDestinationPage/SelectDestinationPage.tsx +++ b/airbyte-webapp/src/pages/destination/SelectDestinationPage/SelectDestinationPage.tsx @@ -8,7 +8,7 @@ import { Box } from "components/ui/Box"; import { Heading } from "components/ui/Heading"; import { useSuggestedDestinations } from "area/connector/utils"; -import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; +import { useDestinationDefinitionList } from "core/api"; export const SelectDestinationPage: React.FC = () => { const navigate = useNavigate(); diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index f4b952271e0..025c5520944 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -4,8 +4,13 @@ import { useEffectOnce } from "react-use"; import { ApiErrorBoundary } from "components/common/ApiErrorBoundary"; -import { useGetInstanceConfiguration, useInvalidateAllWorkspaceScopeOnChange, useListWorkspaces } from "core/api"; +import { + useGetInstanceConfiguration, + useInvalidateAllWorkspaceScopeOnChange, + useListWorkspacesInfinite, +} from "core/api"; import { useAnalyticsIdentifyUser, useAnalyticsRegisterValues } from "core/services/analytics"; +import { useAuthService } from "core/services/auth"; import { FeatureItem, useFeature } from "core/services/features"; import { storeUtmFromQuery } from "core/utils/utmStorage"; import { useApiHealthPoll } from "hooks/services/Health"; @@ -17,6 +22,7 @@ import MainView from "views/layout/MainView"; import { RoutePaths, DestinationPaths, SourcePaths } from "./routePaths"; import { WorkspaceRead } from "../core/request/AirbyteClient"; +const DefaultView = React.lazy(() => import("./DefaultView")); const ConnectionsRoutes = React.lazy(() => import("./connections/ConnectionsRoutes")); const ConnectorBuilderRoutes = React.lazy(() => import("./connectorBuilder/ConnectorBuilderRoutes")); const AllDestinationsPage = React.lazy(() => import("./destination/AllDestinationsPage")); @@ -91,14 +97,19 @@ const PreferencesRoutes = () => ( export const AutoSelectFirstWorkspace: React.FC = () => { const location = useLocation(); - const { workspaces } = useListWorkspaces(); - const currentWorkspace = workspaces[0]; + const { data: workspacesData } = useListWorkspacesInfinite(2, "", true); + const workspaces = workspacesData?.pages?.[0]?.data.workspaces ?? []; + + const currentWorkspace = workspaces.length ? workspaces[0] : undefined; + const [searchParams] = useSearchParams(); return ( { }; const RoutingWithWorkspace: React.FC<{ element?: JSX.Element }> = ({ element }) => { - const { initialSetupComplete } = useGetInstanceConfiguration(); const workspace = useCurrentWorkspace(); + useAddAnalyticsContextForWorkspace(workspace); useApiHealthPoll(); // invalidate everything in the workspace scope when the workspaceId changes useInvalidateAllWorkspaceScopeOnChange(workspace.workspaceId); - return initialSetupComplete ? element ?? : ; + return element ?? ; }; export const Routing: React.FC = () => { + const { inited, user } = useAuthService(); + useBuildUpdateCheck(); const { search } = useLocation(); @@ -126,22 +139,26 @@ export const Routing: React.FC = () => { storeUtmFromQuery(search); }); - // TODO: Remove this after it is verified there are no problems with current routing - const OldRoutes = useMemo( - () => - Object.values(RoutePaths).map((r) => } />), - [] - ); + const multiWorkspaceUI = useFeature(FeatureItem.MultiWorkspaceUI); + const { initialSetupComplete } = useGetInstanceConfiguration(); - const isNewWorkspacesUIEnabled = useFeature(FeatureItem.MultiWorkspaceUI); + if (!inited) { + return null; + } return ( - {OldRoutes} } /> - {isNewWorkspacesUIEnabled && } />} - } /> - } /> + {user && !initialSetupComplete ? ( + } /> + ) : ( + <> + {multiWorkspaceUI && } />} + } /> + } /> + } /> + + )} ); }; diff --git a/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx b/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx index de4546c1af2..51284035e67 100644 --- a/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx +++ b/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx @@ -1,5 +1,3 @@ -import { faPlus } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import { FormattedMessage } from "react-intl"; import { Navigate, useNavigate } from "react-router-dom"; @@ -8,6 +6,7 @@ import { HeadTitle } from "components/common/HeadTitle"; import { MainPageWithScroll } from "components/common/MainPageWithScroll"; import { Button } from "components/ui/Button"; import { Heading } from "components/ui/Heading"; +import { Icon } from "components/ui/Icon"; import { PageHeader } from "components/ui/PageHeader"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; @@ -34,7 +33,7 @@ const AllSourcesPage: React.FC = () => { } endComponent={ - } diff --git a/airbyte-webapp/src/pages/source/CreateSourcePage/CreateSourcePage.tsx b/airbyte-webapp/src/pages/source/CreateSourcePage/CreateSourcePage.tsx index 82cbe56d775..ed2fbebc482 100644 --- a/airbyte-webapp/src/pages/source/CreateSourcePage/CreateSourcePage.tsx +++ b/airbyte-webapp/src/pages/source/CreateSourcePage/CreateSourcePage.tsx @@ -12,12 +12,11 @@ import { Icon } from "components/ui/Icon"; import { PageHeaderWithNavigation } from "components/ui/PageHeader"; import { ConnectionConfiguration } from "area/connector/types"; +import { useSourceDefinitionList } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { useCreateSource } from "hooks/services/useSourceHook"; -import { SourcePaths } from "pages/routePaths"; -import { RoutePaths } from "pages/routePaths"; -import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; +import { SourcePaths, RoutePaths } from "pages/routePaths"; import { ConnectorDocumentationWrapper } from "views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationWrapper"; import { SourceForm } from "./SourceForm"; diff --git a/airbyte-webapp/src/pages/source/SelectSourcePage/SelectSourcePage.tsx b/airbyte-webapp/src/pages/source/SelectSourcePage/SelectSourcePage.tsx index 92eb35864d6..96783406d21 100644 --- a/airbyte-webapp/src/pages/source/SelectSourcePage/SelectSourcePage.tsx +++ b/airbyte-webapp/src/pages/source/SelectSourcePage/SelectSourcePage.tsx @@ -8,7 +8,7 @@ import { Box } from "components/ui/Box"; import { Heading } from "components/ui/Heading"; import { useSuggestedSources } from "area/connector/utils"; -import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; +import { useSourceDefinitionList } from "core/api"; export const SelectSourcePage: React.FC = () => { const navigate = useNavigate(); diff --git a/airbyte-webapp/src/pages/source/SourceItemPage/SourceItemPage.tsx b/airbyte-webapp/src/pages/source/SourceItemPage/SourceItemPage.tsx index 14f1235e276..f920e443449 100644 --- a/airbyte-webapp/src/pages/source/SourceItemPage/SourceItemPage.tsx +++ b/airbyte-webapp/src/pages/source/SourceItemPage/SourceItemPage.tsx @@ -11,11 +11,10 @@ import LoadingPage from "components/LoadingPage"; import { PageHeaderWithNavigation } from "components/ui/PageHeader"; import { useGetSourceFromParams } from "area/connector/utils"; -import { useSourceDefinitionVersion } from "core/api"; +import { useSourceDefinitionVersion, useSourceDefinition } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { RoutePaths } from "pages/routePaths"; -import { useSourceDefinition } from "services/connector/SourceDefinitionService"; import { ResourceNotFoundErrorBoundary } from "views/common/ResourceNotFoundErrorBoundary"; import { StartOverErrorView } from "views/common/StartOverErrorView"; import { ConnectorDocumentationWrapper } from "views/Connector/ConnectorDocumentationLayout"; diff --git a/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx b/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx index 31dc55d1a50..a4cefc3bdff 100644 --- a/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx +++ b/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx @@ -5,12 +5,16 @@ import { Box } from "components/ui/Box"; import { Text } from "components/ui/Text"; import { useGetSourceFromParams } from "area/connector/utils"; -import { useConnectionList, useSourceDefinitionVersion, useGetSourceDefinitionSpecification } from "core/api"; +import { + useConnectionList, + useSourceDefinitionVersion, + useGetSourceDefinitionSpecification, + useSourceDefinition, +} from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { useDeleteSource, useInvalidateSource, useUpdateSource } from "hooks/services/useSourceHook"; import { useDeleteModal } from "hooks/useDeleteModal"; -import { useSourceDefinition } from "services/connector/SourceDefinitionService"; import { ConnectorCard } from "views/Connector/ConnectorCard"; import { ConnectorCardValues } from "views/Connector/ConnectorForm"; diff --git a/airbyte-webapp/src/pages/workspaces/WorkspacesPage.tsx b/airbyte-webapp/src/pages/workspaces/WorkspacesPage.tsx index 3a21c996467..aaaab86b744 100644 --- a/airbyte-webapp/src/pages/workspaces/WorkspacesPage.tsx +++ b/airbyte-webapp/src/pages/workspaces/WorkspacesPage.tsx @@ -15,10 +15,12 @@ import { SearchInput } from "components/ui/SearchInput"; import { Text } from "components/ui/Text"; import { InfoTooltip } from "components/ui/Tooltip"; +import { NoWorkspacePermissionsContent } from "area/workspace/NoWorkspacesPermissionWarning"; import { useCreateWorkspace, useListWorkspacesInfinite } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useAuthService } from "core/services/auth"; +import { useOrganizationsToCreateWorkspaces } from "./components/useOrganizationsToCreateWorkspaces"; import { WorkspacesCreateControl } from "./components/WorkspacesCreateControl"; import WorkspacesList from "./components/WorkspacesList"; import styles from "./WorkspacesPage.module.scss"; @@ -26,9 +28,11 @@ import styles from "./WorkspacesPage.module.scss"; export const WORKSPACE_LIST_LENGTH = 50; const WorkspacesPage: React.FC = () => { - const { isLoading, mutateAsync: handleLogout } = useMutation(() => logout?.() ?? Promise.resolve()); + const { isLoading: isLogoutLoading, mutateAsync: handleLogout } = useMutation(() => logout?.() ?? Promise.resolve()); useTrackPage(PageTrackingCodes.WORKSPACES); const [searchValue, setSearchValue] = useState(""); + const [isSearchEmpty, setIsSearchEmpty] = useState(true); + const { organizationsMemberOnly, organizationsToCreateIn } = useOrganizationsToCreateWorkspaces(); const [debouncedSearchValue, setDebouncedSearchValue] = useState(""); const { @@ -36,6 +40,8 @@ const WorkspacesPage: React.FC = () => { hasNextPage, fetchNextPage, isFetchingNextPage, + isFetching, + isLoading, } = useListWorkspacesInfinite(WORKSPACE_LIST_LENGTH, debouncedSearchValue); const workspaces = workspacesData?.pages.flatMap((page) => page.data.workspaces) ?? []; @@ -43,9 +49,12 @@ const WorkspacesPage: React.FC = () => { const { mutateAsync: createWorkspace } = useCreateWorkspace(); const { logout } = useAuthService(); + const showNoWorkspacesContent = !isFetching && !organizationsToCreateIn.length && !workspaces.length && isSearchEmpty; + useDebounce( () => { setDebouncedSearchValue(searchValue); + setIsSearchEmpty(searchValue === ""); }, 250, [searchValue] @@ -58,7 +67,7 @@ const WorkspacesPage: React.FC = () => { {logout && ( - )} @@ -81,17 +90,28 @@ const WorkspacesPage: React.FC = () => { } /> - - setSearchValue(e.target.value)} /> - - - - - - {isFetchingNextPage && ( - - - + {showNoWorkspacesContent ? ( + + ) : ( + <> + + setSearchValue(e.target.value)} /> + + + + + + {isFetchingNextPage && ( + + + + )} + )} diff --git a/airbyte-webapp/src/pages/workspaces/components/WorkspacesCreateControl.tsx b/airbyte-webapp/src/pages/workspaces/components/WorkspacesCreateControl.tsx index d5085eb8df8..fdfa5a0636c 100644 --- a/airbyte-webapp/src/pages/workspaces/components/WorkspacesCreateControl.tsx +++ b/airbyte-webapp/src/pages/workspaces/components/WorkspacesCreateControl.tsx @@ -1,14 +1,9 @@ /** - * As written, this workspace create control can ONLY be used in environments - * where the configdb Permissions table is in use. - * - * That is to say -- it should currently only be used in OSS/Enterprise. - * - * May be migrated to Cloud when: - * - Cloud leverages the configdb Permissions table - * - Cloud has a concept of organizations - * - CloudWorkspaceCreate accepts an organizationId - * + * This should not be used in cloud until: + * - all cloud users have an org + * - all cloud users have permissions in configdb + * - we are able to use the oss create workspace endpoint in cloud + */ import { UseMutateAsyncFunction } from "@tanstack/react-query"; diff --git a/airbyte-webapp/src/pages/workspaces/components/WorkspacesList.tsx b/airbyte-webapp/src/pages/workspaces/components/WorkspacesList.tsx index 0d940ff2570..a9a3bde3809 100644 --- a/airbyte-webapp/src/pages/workspaces/components/WorkspacesList.tsx +++ b/airbyte-webapp/src/pages/workspaces/components/WorkspacesList.tsx @@ -5,6 +5,7 @@ import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; +import { LoadingSpinner } from "components/ui/LoadingSpinner"; import { CloudWorkspaceRead } from "core/api/types/CloudApi"; import { WorkspaceRead } from "core/request/AirbyteClient"; @@ -15,8 +16,14 @@ interface WorkspacesListProps { workspaces: WorkspaceRead[] | CloudWorkspaceRead[]; fetchNextPage: () => void; hasNextPage?: boolean; + isLoading?: boolean; } -export const WorkspacesList: React.FC = ({ workspaces, fetchNextPage, hasNextPage }) => { +export const WorkspacesList: React.FC = ({ + workspaces, + fetchNextPage, + hasNextPage, + isLoading, +}) => { const { ref, inView } = useInView(); useEffect(() => { @@ -25,6 +32,16 @@ export const WorkspacesList: React.FC = ({ workspaces, fetc } }, [inView, fetchNextPage, hasNextPage]); + if (isLoading) { + return ( + + + + + + ); + } + return ( {workspaces.length ? ( diff --git a/airbyte-webapp/src/scss/_z-indices.scss b/airbyte-webapp/src/scss/_z-indices.scss index 873c5efbdd1..de201ff09b7 100644 --- a/airbyte-webapp/src/scss/_z-indices.scss +++ b/airbyte-webapp/src/scss/_z-indices.scss @@ -1,6 +1,7 @@ $base: 0; $bulkEdit: 1000; $sidebar: 9999; +$deployPreviewMessage: $sidebar + 6; $workspacePicker: $sidebar + 5; $tooltip: $sidebar + 4; $notification: $sidebar + 3; diff --git a/airbyte-webapp/src/test-utils/mock-data/mockSvg.js b/airbyte-webapp/src/test-utils/mock-data/mockSvg.js index e972d17470a..6e41c3880c2 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockSvg.js +++ b/airbyte-webapp/src/test-utils/mock-data/mockSvg.js @@ -2,4 +2,6 @@ // https://react-svgr.com/docs/jest/ // eslint-disable-next-line import/no-anonymous-default-export -export default "div"; +const MockSvg = (props) =>
; + +export default MockSvg; diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.module.scss b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.module.scss index 86c1813d399..ee85245b3f3 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.module.scss +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.module.scss @@ -19,4 +19,5 @@ .collapsibleIpAddresses { gap: variables.$spacing-md; + font-size: variables.$font-size-md; } diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestCard.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestCard.tsx index fa8811383e6..9697e4a2d9a 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestCard.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestCard.tsx @@ -1,5 +1,3 @@ -import { faClose, faRefresh } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import { FormattedMessage } from "react-intl"; @@ -7,6 +5,7 @@ import { JobFailure } from "components/JobFailure"; import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; import { ProgressBar } from "components/ui/ProgressBar"; import { Text } from "components/ui/Text"; @@ -64,7 +63,7 @@ export const TestCard: React.FC = ({ {isTestConnectionInProgress || !isEditMode ? (
{diffType === "add" ? ( - + ) : diffType === "remove" ? ( - + ) : (
- +
)}
@@ -65,12 +62,7 @@ export const FieldRow: React.FC = ({ transform }) => {
- } + control={} > @@ -82,7 +74,7 @@ export const FieldRow: React.FC = ({ transform }) => {
- {oldType} {newType} + {oldType} {newType}