diff --git a/.editorconfig b/.editorconfig index 73b614047e4..7de29a1dfa8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -23,6 +23,11 @@ ij_json_spaces_within_braces = false ij_json_spaces_within_brackets = false ij_json_wrap_long_lines = false +[*.{kt,kts}] +indent_size = 2 +max_line_length = 150 +ij_kotlin_packages_to_use_import_on_demand = unset + [{*.markdown,*.md}] ij_markdown_force_one_space_after_blockquote_symbol = true ij_markdown_force_one_space_after_header_symbol = true diff --git a/.env b/.env index 97b349f0435..d0a3e41a012 100644 --- a/.env +++ b/.env @@ -26,7 +26,7 @@ WORKSPACE_DOCKER_MOUNT=airbyte_workspace # be the same as *_ROOT. # Issue: https://github.com/airbytehq/airbyte/issues/578 LOCAL_ROOT=/tmp/airbyte_local -LOCAL_DOCKER_MOUNT=/tmp/airbyte_local +LOCAL_DOCKER_MOUNT=oss_local_root # todo (cgardens) - hack to handle behavior change in docker compose. *_PARENT directories MUST # already exist on the host filesystem and MUST be parents of *_ROOT. # Issue: https://github.com/airbytehq/airbyte/issues/577 @@ -65,17 +65,17 @@ CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.40.23.002 TEMPORAL_HOST=airbyte-temporal:7233 INTERNAL_API_HOST=airbyte-server:8001 INTERNAL_API_URL=http://airbyte-server:8001 -CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80 +CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:8080 WEBAPP_URL=http://localhost:8000/ WORKLOAD_API_HOST=workload-api-server:8007 WORKLOAD_API_URL=http://workload-api-server:8007 # Although not present as an env var, required for webapp configuration. CONNECTOR_BUILDER_API_URL=/connector-builder-api AIRBYTE_API_HOST=airbyte-api-server:8006 -CONNECTOR_BUILDER_SERVER_API_HOST=http://airbyte-connector-builder-server:80 +CONNECTOR_BUILDER_SERVER_API_HOST=http://airbyte-connector-builder-server:8080 # Replace with the commented-out line below to use a locally-run connector-builder-server # image, e.g. when developing the CDK's builder server command runner. -# CONNECTOR_BUILDER_SERVER_API_HOST=http://host.docker.internal:80 +# CONNECTOR_BUILDER_SERVER_API_HOST=http://host.docker.internal:8080 ### JOBS ### # Relevant to scaling. diff --git a/.github/actions/match-github-to-slack-user/action.yml b/.github/actions/match-github-to-slack-user/action.yml new file mode 100644 index 00000000000..90fff4ddbd0 --- /dev/null +++ b/.github/actions/match-github-to-slack-user/action.yml @@ -0,0 +1,25 @@ +# This action will try to match git commit author (GITHUB_ACTOR) with Slack user +# and add it to GITHUB_OUTPUT +# Following env variables should be provided. +# Provided by Github: +# GITHUB_ACTOR: commit author +# GITHUB_REPOSITORY: name of the repo we check the commit author, e.g. "airbytehq/airbyte-platform-internal" +# Required: +# AIRBYTE_HQ_BOT_SLACK_TOKEN: ${{ secrets.AIRBYTE_HQ_BOT_SLACK_TOKEN }} +# AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.AIRBYTE_TEAM_BOT_SLACK_TOKEN }} +# GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +name: 'Match Github user to Slack user' +description: 'Match Github user to Slack by email or full name in Github profile.' +outputs: + slack_user_ids: + description: 'Comma separated slack user IDs that match to GITHUB_ACTOR (Github username)' + value: ${{ steps.match-github-to-slack-user.outputs.slack_user_ids }} +runs: + using: 'composite' + steps: + - name: Match github user to slack user + id: match-github-to-slack-user + run: | + ./tools/bin/match_github_user_to_slack + shell: bash diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a5e02d08d78..87cbb220b03 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,22 +1,23 @@ ## What -*Describe what the change is solving* -*It helps to add screenshots if it affects the frontend.* + ## How -*Describe the solution* + ## Recommended reading order -1. `x.java` -2. `y.java` +1. `x.kt` +2. `y.kt` -## Can this PR be safely reverted / rolled back? -*If you know that your PR is backwards-compatible and can be simply reverted or rolled back, check the YES box.* - -*Otherwise if your PR has a breaking change, like a database migration for example, check the NO box.* - -*If unsure, leave it blank.* +## Can this PR be safely reverted and rolled back? + - [ ] YES 💚 - [ ] NO ❌ - -## 🚨 User Impact 🚨 -Are there any breaking changes? What is the end result perceived by the user? If yes, please merge this PR with the 🚨🚨 emoji so changelog authors can further highlight this if needed. diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index dcc9e372a6b..61b4b029f7e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1,4 +1,7 @@ -name: Airbyte Platform CI +# The goal of this build is to make sure that OSS contributors can build the project and run the tests +# so that they can develop locally. It is NOT a release verification. As such, we just run build and +# unit test. No additional acceptance test, etc. +name: Airbyte Platform OSS Developer Build env: S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} @@ -24,7 +27,6 @@ on: permissions: write-all jobs: - # COMMON TASKS ensure-images-exist: name: "Ensure all required Docker images exist on Dockerhub" timeout-minutes: 10 @@ -94,275 +96,18 @@ jobs: # - run: | # echo '${{ toJSON(needs) }}' - ## BUILDS - ## Frontend Test - # In case of self-hosted EC2 errors, remove this block. - start-frontend-runner: - name: "Frontend: Start EC2 Runner" - needs: - - changes - # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on main. - if: | - needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/main' - || (always() && needs.changes.outputs.backend == 'true') - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ env.PAT }} - frontend-build: - name: "Frontend: Build" - needs: - - start-frontend-runner - runs-on: ${{ needs.start-frontend-runner.outputs.label }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - # We need to fetch at least one more commmit for the Chromatic action not to fail - # but since we don't do screenshot comparison we don't need to fetch the full history. - with: - fetch-depth: 2 - - - name: Cache Build Artifacts - uses: ./.github/actions/cache-build-artifacts - with: - cache-key: ${{ secrets.CACHE_VERSION }} - cache-python: "false" - - - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "21" - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Set up CI Gradle Properties - run: | - mkdir -p ~/.gradle/ - cat > ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties <?, + metadata: Map, ) } @@ -106,26 +105,23 @@ class SegmentTrackingClient( override fun identify(workspaceId: UUID) { val deployment: Deployment = deploymentFetcher.get() val trackingIdentity: TrackingIdentity = trackingIdentityFetcher.apply(workspaceId) - val identityMetadata: MutableMap = HashMap() - - // deployment - identityMetadata[AIRBYTE_VERSION_KEY] = deployment.getDeploymentVersion() - identityMetadata["deployment_mode"] = deployment.getDeploymentMode() - identityMetadata["deployment_env"] = deployment.getDeploymentEnvironment() - identityMetadata["deployment_id"] = deployment.getDeploymentId().toString() - - // workspace (includes info that in the future we would store in an organization) - identityMetadata["anonymized"] = trackingIdentity.isAnonymousDataCollection() - identityMetadata["subscribed_newsletter"] = trackingIdentity.isNews() - identityMetadata["subscribed_security"] = trackingIdentity.isSecurityUpdates() - if (trackingIdentity.email != null) { - identityMetadata["email"] = trackingIdentity.email - } - - // other - if (airbyteRole.isNotBlank()) { - identityMetadata[AIRBYTE_ROLE] = airbyteRole - } + val identityMetadata: Map = + buildMap { + // deployment + put(AIRBYTE_VERSION_KEY, deployment.getDeploymentVersion()) + put("deployment_mode", deployment.getDeploymentMode()) + put("deployment_env", deployment.getDeploymentEnvironment()) + put("deployment_id", deployment.getDeploymentId().toString()) + + // workspace (includes info that in the future we would store in an organization) + put("anonymized", trackingIdentity.isAnonymousDataCollection()) + put("subscribed_newsletter", trackingIdentity.isNews()) + put("subscribed_security", trackingIdentity.isSecurityUpdates()) + trackingIdentity.email?.let { put("email", it) } + + // other + airbyteRole.takeIf { it.isNotBlank() }?.let { put(AIRBYTE_ROLE, it) } + } val joinKey: String = trackingIdentity.customerId.toString() segmentAnalyticsClient.analyticsClient.enqueue( @@ -147,32 +143,34 @@ class SegmentTrackingClient( workspaceId: UUID, action: String?, ) { - track(workspaceId, action, emptyMap()) + track(workspaceId, action, emptyMap()) } override fun track( workspaceId: UUID, action: String?, - metadata: Map?, + metadata: Map, ) { - val mapCopy: MutableMap = java.util.HashMap(metadata) val deployment: Deployment = deploymentFetcher.get() val trackingIdentity: TrackingIdentity = trackingIdentityFetcher.apply(workspaceId) - val airbyteSource: Optional = getAirbyteSource() - mapCopy[AIRBYTE_SOURCE] = airbyteSource.orElse(UNKNOWN) - - // Always add these traits. - mapCopy[AIRBYTE_VERSION_KEY] = deployment.getDeploymentVersion() - mapCopy[CUSTOMER_ID_KEY] = trackingIdentity.customerId - mapCopy[AIRBYTE_DEPLOYMENT_ID] = deployment.getDeploymentId().toString() - mapCopy[AIRBYTE_DEPLOYMENT_MODE] = deployment.getDeploymentMode() - mapCopy[AIRBYTE_TRACKED_AT] = Instant.now().toString() - if (metadata!!.isNotEmpty()) { - if (trackingIdentity.email != null) { - mapCopy["email"] = trackingIdentity.email + val mapCopy: Map = + buildMap { + putAll(metadata) + put(AIRBYTE_SOURCE, getAirbyteSource() ?: UNKNOWN) + + // Always add these traits. + put(AIRBYTE_VERSION_KEY, deployment.getDeploymentVersion()) + put(CUSTOMER_ID_KEY, trackingIdentity.customerId) + put(AIRBYTE_DEPLOYMENT_ID, deployment.getDeploymentId().toString()) + put(AIRBYTE_DEPLOYMENT_MODE, deployment.getDeploymentMode()) + put(AIRBYTE_TRACKED_AT, Instant.now().toString()) + if (metadata.isNotEmpty()) { + if (trackingIdentity.email != null) { + put("email", trackingIdentity.email) + } + } } - } val joinKey: String = trackingIdentity.customerId.toString() segmentAnalyticsClient.analyticsClient.enqueue( @@ -182,12 +180,12 @@ class SegmentTrackingClient( ) } - private fun getAirbyteSource(): Optional { + private fun getAirbyteSource(): String? { val currentRequest = ServerRequestContext.currentRequest() return if (currentRequest.isPresent) { - Optional.ofNullable(currentRequest.get().headers[AIRBYTE_ANALYTIC_SOURCE_HEADER]) + currentRequest.get().headers[AIRBYTE_ANALYTIC_SOURCE_HEADER] } else { - Optional.empty() + null } } @@ -328,7 +326,7 @@ class LoggingTrackingClient( override fun track( workspaceId: UUID, action: String?, - metadata: Map?, + metadata: Map, ) { val deployment: Deployment = deploymentFetcher.get() val trackingIdentity: TrackingIdentity = trackingIdentityFetcher.apply(workspaceId) diff --git a/airbyte-analytics/src/test/kotlin/io/airbyte/analytics/SegmentTrackingClientTest.kt b/airbyte-analytics/src/test/kotlin/io/airbyte/analytics/SegmentTrackingClientTest.kt index 9d95a8ce32f..2f0e0362c42 100644 --- a/airbyte-analytics/src/test/kotlin/io/airbyte/analytics/SegmentTrackingClientTest.kt +++ b/airbyte-analytics/src/test/kotlin/io/airbyte/analytics/SegmentTrackingClientTest.kt @@ -103,7 +103,7 @@ class SegmentTrackingClientTest { verify(exactly = 1) { analytics.enqueue(any()) } val actual = builderSlot.captured.build() - val expectedTraits: Map? = + val expectedTraits: Map = mapOf( "airbyte_role" to "role", SegmentTrackingClient.AIRBYTE_VERSION_KEY to airbyteVersion.serialize(), @@ -124,7 +124,7 @@ class SegmentTrackingClientTest { val builderSlot = slot() every { analytics.enqueue(capture(builderSlot)) } returns Unit - val metadata: Map? = + val metadata: Map = mapOf( SegmentTrackingClient.AIRBYTE_VERSION_KEY to airbyteVersion.serialize(), "user_id" to identity.customerId, @@ -147,7 +147,7 @@ class SegmentTrackingClientTest { val builderSlot = slot() every { analytics.enqueue(capture(builderSlot)) } returns Unit - val metadata: Map? = + val metadata: Map = mapOf( SegmentTrackingClient.AIRBYTE_VERSION_KEY to airbyteVersion.serialize(), EMAIL_KEY to EMAIL, @@ -177,7 +177,7 @@ class SegmentTrackingClientTest { every { httpRequest.headers } returns httpHeaders ServerRequestContext.with(httpRequest) { - val metadata: Map? = + val metadata: Map = mapOf( SegmentTrackingClient.AIRBYTE_VERSION_KEY to airbyteVersion.serialize(), EMAIL_KEY to EMAIL, diff --git a/airbyte-api-server/Dockerfile b/airbyte-api-server/Dockerfile index 2a2c22b19db..732938151a7 100644 --- a/airbyte-api-server/Dockerfile +++ b/airbyte-api-server/Dockerfile @@ -1,12 +1,16 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.0.1 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.1.0 FROM ${JDK_IMAGE} AS server EXPOSE 8006 5005 ENV APPLICATION airbyte-api-server ENV VERSION ${VERSION} + WORKDIR /app # This is automatically unzipped by Docker +USER root ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"] diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingHelper.kt index 06f03e247af..52facb295c7 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingHelper.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingHelper.kt @@ -131,7 +131,7 @@ class TrackingHelper(private val trackingClient: TrackingClient) { trackingClient.track( userId, AIRBYTE_API_CALL, - payload as Map?, + payload.toMap(), ) } diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java index ae4cb5134a4..21066fb7164 100644 --- a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java +++ b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java @@ -64,7 +64,6 @@ public class AirbyteApiClient { private final DestinationDefinitionSpecificationApi destinationSpecificationApi; private final JobsApi jobsApi; private final JobRetryStatesApi jobRetryStatesApi; - private final PatchedLogsApi logsApi; private final OperationApi operationApi; private final SourceDefinitionApi sourceDefinitionApi; private final SourceApi sourceApi; @@ -87,7 +86,6 @@ public AirbyteApiClient(final ApiClient apiClient) { destinationSpecificationApi = new DestinationDefinitionSpecificationApi(apiClient); jobsApi = new JobsApi(apiClient); jobRetryStatesApi = new JobRetryStatesApi(apiClient); - logsApi = new PatchedLogsApi(apiClient); operationApi = new OperationApi(apiClient); sourceDefinitionApi = new SourceDefinitionApi(apiClient); sourceApi = new SourceApi(apiClient); @@ -147,10 +145,6 @@ public WorkspaceApi getWorkspaceApi() { return workspaceApi; } - public PatchedLogsApi getLogsApi() { - return logsApi; - } - public OperationApi getOperationApi() { return operationApi; } diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java b/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java deleted file mode 100644 index 9a2aaf2061b..00000000000 --- a/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.api.client; - -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.api.client.invoker.generated.ApiClient; -import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.api.client.invoker.generated.ApiResponse; -import io.airbyte.api.client.model.generated.LogsRequestBody; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.function.Consumer; -import org.apache.commons.io.FileUtils; - -/** - * This class is a copy of {@link io.airbyte.api.client.generated.LogsApi} except it allows Accept: - * text/plain. Without this modification, {@link io.airbyte.api.client.generated.LogsApi} returns a - * 406 because the generated code requests the wrong response type. - */ -public class PatchedLogsApi { - - private final HttpClient memberVarHttpClient; - private final ObjectMapper memberVarObjectMapper; - private final String memberVarBaseUri; - private final Consumer memberVarInterceptor; - private final Duration memberVarReadTimeout; - private final Consumer> memberVarResponseInterceptor; - - public PatchedLogsApi() { - this(new ApiClient()); - } - - public PatchedLogsApi(final ApiClient apiClient) { - memberVarHttpClient = apiClient.getHttpClient(); - memberVarObjectMapper = apiClient.getObjectMapper(); - memberVarBaseUri = apiClient.getBaseUri(); - memberVarInterceptor = apiClient.getRequestInterceptor(); - memberVarReadTimeout = apiClient.getReadTimeout(); - memberVarResponseInterceptor = apiClient.getResponseInterceptor(); - } - - /** - * Get logs. - * - * @param logsRequestBody (required) - * @return File - * @throws ApiException if fails to make API call - */ - public File getLogs(final LogsRequestBody logsRequestBody) throws ApiException { - final ApiResponse localVarResponse = getLogsWithHttpInfo(logsRequestBody); - return localVarResponse.getData(); - } - - /** - * Get logs. - * - * @param logsRequestBody (required) - * @return ApiResponse<File> - * @throws ApiException if fails to make API call - */ - public ApiResponse getLogsWithHttpInfo(final LogsRequestBody logsRequestBody) throws ApiException { - final HttpRequest.Builder localVarRequestBuilder = getLogsRequestBuilder(logsRequestBody); - try { - final HttpResponse localVarResponse = memberVarHttpClient.send( - localVarRequestBuilder.build(), - HttpResponse.BodyHandlers.ofInputStream()); - if (memberVarResponseInterceptor != null) { - memberVarResponseInterceptor.accept(localVarResponse); - } - if (isErrorResponse(localVarResponse)) { - throw new ApiException(localVarResponse.statusCode(), - "getLogs call received non-success response", - localVarResponse.headers(), - localVarResponse.body() == null ? null : new String(localVarResponse.body().readAllBytes())); - } - - final File tmpFile = File.createTempFile("patched-logs-api", "response"); // CHANGED - tmpFile.deleteOnExit(); // CHANGED - - FileUtils.copyInputStreamToFile(localVarResponse.body(), tmpFile); // CHANGED - - return new ApiResponse( - localVarResponse.statusCode(), - localVarResponse.headers().map(), - tmpFile // CHANGED - ); - } catch (final IOException e) { - throw new ApiException(e); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ApiException(e); - } - } - - private Boolean isErrorResponse(final HttpResponse httpResponse) { - return httpResponse.statusCode() / 100 != 2; - } - - private HttpRequest.Builder getLogsRequestBuilder(final LogsRequestBody logsRequestBody) throws ApiException { - // verify the required parameter 'logsRequestBody' is set - if (logsRequestBody == null) { - throw new ApiException(400, "Missing the required parameter 'logsRequestBody' when calling getLogs"); - } - - final HttpRequest.Builder localVarRequestBuilder = HttpRequest.newBuilder(); - - final String localVarPath = "/v1/logs/get"; - - localVarRequestBuilder.uri(URI.create(memberVarBaseUri + localVarPath)); - - localVarRequestBuilder.header("Content-Type", "application/json"); - - localVarRequestBuilder.header("Accept", "text/plain"); // CHANGED - - try { - final byte[] localVarPostBody = memberVarObjectMapper.writeValueAsBytes(logsRequestBody); - localVarRequestBuilder.method("POST", HttpRequest.BodyPublishers.ofByteArray(localVarPostBody)); - } catch (final IOException e) { - throw new ApiException(e); - } - if (memberVarReadTimeout != null) { - localVarRequestBuilder.timeout(memberVarReadTimeout); - } - if (memberVarInterceptor != null) { - memberVarInterceptor.accept(localVarRequestBuilder); - } - return localVarRequestBuilder; - } - -} diff --git a/airbyte-api/src/main/kotlin/AirbyteApiClient2.kt b/airbyte-api/src/main/kotlin/AirbyteApiClient2.kt index ace9f4a9c1e..daa46d0b2d1 100644 --- a/airbyte-api/src/main/kotlin/AirbyteApiClient2.kt +++ b/airbyte-api/src/main/kotlin/AirbyteApiClient2.kt @@ -16,6 +16,7 @@ import io.airbyte.api.client2.generated.HealthApi import io.airbyte.api.client2.generated.JobRetryStatesApi import io.airbyte.api.client2.generated.JobsApi import io.airbyte.api.client2.generated.OperationApi +import io.airbyte.api.client2.generated.PermissionApi import io.airbyte.api.client2.generated.SecretsPersistenceConfigApi import io.airbyte.api.client2.generated.SourceApi import io.airbyte.api.client2.generated.SourceDefinitionApi @@ -83,6 +84,7 @@ class AirbyteApiClient2 val stateApi = StateApi(basePath = basePath, client = httpClient, policy = policy) val streamStatusesApi = StreamStatusesApi(basePath = basePath, client = httpClient, policy = policy) val secretPersistenceConfigApi = SecretsPersistenceConfigApi(basePath = basePath, client = httpClient, policy = policy) + val permissonsApi = PermissionApi(basePath = basePath, client = httpClient, policy = policy) } class ThrowOn5xxInterceptor : okhttp3.Interceptor { diff --git a/airbyte-api/src/main/openapi/cloud-config.yaml b/airbyte-api/src/main/openapi/cloud-config.yaml index 7e9b002e3e3..3a21f899d51 100644 --- a/airbyte-api/src/main/openapi/cloud-config.yaml +++ b/airbyte-api/src/main/openapi/cloud-config.yaml @@ -213,6 +213,25 @@ paths: $ref: "#/components/schemas/CreateKeycloakUserResponseBody" "409": $ref: "#/components/responses/ExceptionResponse" + /v1/users/send_verification_email: + post: + tags: + - user + summary: Triggers a verification email to be sent to the user + operationId: sendVerificationEmail + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/UserIdRequestBody" + required: true + responses: + "204": + description: The verification email was sent successfully. + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" # CLOUD_WORKSPACE /v1/cloud_workspaces/create: post: diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 43aa8e142fc..093caafe63c 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -313,6 +313,29 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/workspaces/get_by_connection_id_with_tombstone: + post: + tags: + - workspace + summary: Find workspace by connection id including the tombstone ones + operationId: getWorkspaceByConnectionIdWithTombstone + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectionIdRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/WorkspaceRead" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/workspaces/get_organization_info: post: tags: @@ -4597,11 +4620,11 @@ paths: $ref: "#/components/schemas/UserInvitationCreateRequestBody" responses: "201": - description: Successfully created user invitation. + description: Successfully processed user invitation create request. content: application/json: schema: - $ref: "#/components/schemas/UserInvitationRead" + $ref: "#/components/schemas/UserInvitationCreateResponse" /v1/user_invitations/by_code/{inviteCode}: get: @@ -6345,6 +6368,8 @@ components: $ref: "#/components/schemas/WebhookConfigRead" organizationId: $ref: "#/components/schemas/OrganizationId" + tombstone: + type: boolean WorkspaceOrganizationInfoRead: type: object description: Limited info about a workspace's organization that is safe to expose to workspace readers who are not members of the org. @@ -11207,6 +11232,15 @@ components: scopeId: type: string format: uuid + UserInvitationCreateResponse: + type: object + properties: + inviteCode: + type: string + description: The created invite code, if the request resulted in a new invitation being created. + directlyAdded: + type: boolean + description: True if the request resulted in the user being directly added, without a created invitation. UserInvitationListRequestBody: type: object required: diff --git a/airbyte-base-java-python-image/Dockerfile b/airbyte-base-java-python-image/Dockerfile deleted file mode 100644 index 245e45060e6..00000000000 --- a/airbyte-base-java-python-image/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM airbyte/airbyte-base-java-image:3.0.1 - -RUN yum update -y && \ - yum groupinstall -y "Development Tools" && \ - yum install -y openssl11-devel bzip2-devel libffi-devel zlib-devel sqlite-devel xz-devel - -ENV PYTHON_VERSION=3.9.11 - -# Set up python -RUN git clone https://github.com/pyenv/pyenv.git ~/.pyenv -ENV PYENV_ROOT /root/.pyenv -ENV PATH ${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:$PATH -RUN pyenv install ${PYTHON_VERSION} && pyenv global ${PYTHON_VERSION} diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile index a856300e575..73531a2a4d2 100644 --- a/airbyte-bootloader/Dockerfile +++ b/airbyte-bootloader/Dockerfile @@ -1,5 +1,11 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.0.1 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.1.0 FROM ${JDK_IMAGE} + WORKDIR /app + +USER root ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte + ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/airbyte-bootloader"] diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index 698fd9b2aae..1a8891f0a96 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -97,8 +97,8 @@ class BootloaderTest { // ⚠️ This line should change with every new migration to show that you meant to make a new // migration to the prod database - private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.41.009"; - private static final String CURRENT_JOBS_MIGRATION_VERSION = "0.50.4.002"; + private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.41.012"; + private static final String CURRENT_JOBS_MIGRATION_VERSION = "0.50.4.003"; private static final String CDK_VERSION = "1.2.3"; @BeforeEach diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/errors/ConflictException.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/errors/ConflictException.java new file mode 100644 index 00000000000..e9925cc276f --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/errors/ConflictException.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.errors; + +import io.micronaut.http.HttpStatus; + +/** + * Exception when a request conflicts with the current state of the server. For example, trying to + * accept an invitation that was already accepted. + */ +public class ConflictException extends KnownException { + + public ConflictException(final String message) { + super(message); + } + + public ConflictException(final String message, final Throwable cause) { + super(message, cause); + } + + @Override + public int getHttpCode() { + return HttpStatus.CONFLICT.getCode(); + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/OAuthHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/OAuthHandler.java index 0b61e10476e..bdc560d2cc1 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/OAuthHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/OAuthHandler.java @@ -262,11 +262,11 @@ public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsent public CompleteOAuthResponse completeSourceOAuthHandleReturnSecret(final CompleteSourceOauthRequest completeSourceOauthRequest) throws JsonValidationException, ConfigNotFoundException, IOException { - final CompleteOAuthResponse oAuthTokens = completeSourceOAuth(completeSourceOauthRequest); - if (oAuthTokens != null && completeSourceOauthRequest.getReturnSecretCoordinate()) { - return writeOAuthResponseSecret(completeSourceOauthRequest.getWorkspaceId(), oAuthTokens); + final CompleteOAuthResponse completeOAuthResponse = completeSourceOAuth(completeSourceOauthRequest); + if (completeOAuthResponse != null && completeSourceOauthRequest.getReturnSecretCoordinate()) { + return writeOAuthResponseSecret(completeSourceOauthRequest.getWorkspaceId(), completeOAuthResponse); } else { - return oAuthTokens; + return completeOAuthResponse; } } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java index da13b8adac6..81aaf96d004 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java @@ -4,6 +4,8 @@ package io.airbyte.commons.server.handlers; +import static io.airbyte.config.persistence.ConfigNotFoundException.NO_ORGANIZATION_FOR_WORKSPACE; + import com.github.slugify.Slugify; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -127,7 +129,8 @@ private static WorkspaceRead buildWorkspaceRead(final StandardWorkspace workspac .notifications(NotificationConverter.toApiList(workspace.getNotifications())) .notificationSettings(NotificationSettingsConverter.toApi(workspace.getNotificationSettings())) .defaultGeography(Enums.convertTo(workspace.getDefaultGeography(), Geography.class)) - .organizationId(workspace.getOrganizationId()); + .organizationId(workspace.getOrganizationId()) + .tombstone(workspace.getTombstone()); // Add read-only webhook configs. if (workspace.getWebhookOperationConfigs() != null) { result.setWebhookConfigs(WorkspaceWebhookConfigsConverter.toApiReads(workspace.getWebhookOperationConfigs())); @@ -349,7 +352,7 @@ public WorkspaceOrganizationInfoRead getWorkspaceOrganizationInfo(final Workspac final UUID workspaceId = workspaceIdRequestBody.getWorkspaceId(); final Optional organization = organizationPersistence.getOrganizationByWorkspaceId(workspaceId); if (organization.isEmpty()) { - throw new ConfigNotFoundException("ORGANIZATION_FOR_WORKSPACE", workspaceId.toString()); + throw new ConfigNotFoundException(NO_ORGANIZATION_FOR_WORKSPACE, workspaceId.toString()); } return buildWorkspaceOrganizationInfoRead(organization.get()); } @@ -361,8 +364,10 @@ public WorkspaceRead getWorkspaceBySlug(final SlugRequestBody slugRequestBody) t return buildWorkspaceRead(workspace); } - public WorkspaceRead getWorkspaceByConnectionId(final ConnectionIdRequestBody connectionIdRequestBody) throws ConfigNotFoundException { - final StandardWorkspace workspace = configRepository.getStandardWorkspaceFromConnection(connectionIdRequestBody.getConnectionId(), false); + public WorkspaceRead getWorkspaceByConnectionId(final ConnectionIdRequestBody connectionIdRequestBody, boolean includeTombstone) + throws ConfigNotFoundException { + final StandardWorkspace workspace = + configRepository.getStandardWorkspaceFromConnection(connectionIdRequestBody.getConnectionId(), includeTombstone); return buildWorkspaceRead(workspace); } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java index 7caf2ca36ee..7c9a20629c2 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java @@ -277,7 +277,8 @@ void testCreateWorkspace() throws JsonValidationException, IOException, ConfigNo .notificationSettings(generateApiNotificationSettingsWithDefaultValue()) .defaultGeography(GEOGRAPHY_US) .webhookConfigs(List.of(new WebhookConfigRead().id(uuid).name(TEST_NAME))) - .organizationId(ORGANIZATION_ID); + .organizationId(ORGANIZATION_ID) + .tombstone(false); assertEquals(expectedRead, actualRead); } @@ -326,7 +327,8 @@ void testCreateWorkspaceIfNotExist() throws JsonValidationException, IOException .notificationSettings(generateApiNotificationSettingsWithDefaultValue()) .defaultGeography(GEOGRAPHY_US) .webhookConfigs(List.of(new WebhookConfigRead().id(uuid).name(TEST_NAME))) - .organizationId(ORGANIZATION_ID); + .organizationId(ORGANIZATION_ID) + .tombstone(false); assertEquals(expectedRead, actualRead); assertEquals(expectedRead, secondActualRead); @@ -360,7 +362,8 @@ void testCreateWorkspaceWithMinimumInput() throws JsonValidationException, IOExc .notifications(List.of()) .notificationSettings(generateDefaultApiNotificationSettings()) .defaultGeography(GEOGRAPHY_AUTO) - .webhookConfigs(Collections.emptyList()); + .webhookConfigs(Collections.emptyList()) + .tombstone(false); assertEquals(expectedRead, actualRead); } @@ -401,7 +404,8 @@ void testCreateWorkspaceDuplicateSlug() throws JsonValidationException, IOExcept .notifications(Collections.emptyList()) .notificationSettings(generateDefaultApiNotificationSettings()) .defaultGeography(GEOGRAPHY_AUTO) - .webhookConfigs(Collections.emptyList()); + .webhookConfigs(Collections.emptyList()) + .tombstone(false); assertTrue(actualRead.getSlug().startsWith(workspace.getSlug())); assertNotEquals(workspace.getSlug(), actualRead.getSlug()); @@ -463,7 +467,8 @@ void testListWorkspaces() throws JsonValidationException, IOException { .notifications(List.of(generateApiNotification())) .notificationSettings(generateApiNotificationSettings()) .defaultGeography(GEOGRAPHY_AUTO) - .organizationId(ORGANIZATION_ID); + .organizationId(ORGANIZATION_ID) + .tombstone(false); final WorkspaceRead expectedWorkspaceRead2 = new WorkspaceRead() .workspaceId(workspace2.getWorkspaceId()) @@ -479,7 +484,8 @@ void testListWorkspaces() throws JsonValidationException, IOException { .notifications(List.of(generateApiNotification())) .notificationSettings(generateApiNotificationSettings()) .defaultGeography(GEOGRAPHY_AUTO) - .organizationId(ORGANIZATION_ID); + .organizationId(ORGANIZATION_ID) + .tombstone(false); final WorkspaceReadList actualWorkspaceReadList = workspacesHandler.listWorkspaces(); @@ -509,31 +515,18 @@ void testGetWorkspace() throws JsonValidationException, ConfigNotFoundException, .notificationSettings(generateApiNotificationSettings()) .defaultGeography(GEOGRAPHY_AUTO) .webhookConfigs(List.of(new WebhookConfigRead().id(WEBHOOK_CONFIG_ID).name(TEST_NAME))) - .organizationId(ORGANIZATION_ID); + .organizationId(ORGANIZATION_ID) + .tombstone(false); assertEquals(workspaceRead, workspacesHandler.getWorkspace(workspaceIdRequestBody)); } @Test - void testGetWorkspaceBySlug() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetWorkspaceBySlug() throws ConfigNotFoundException, IOException { when(configRepository.getWorkspaceBySlug("default", false)).thenReturn(workspace); final SlugRequestBody slugRequestBody = new SlugRequestBody().slug("default"); - final WorkspaceRead workspaceRead = new WorkspaceRead() - .workspaceId(workspace.getWorkspaceId()) - .customerId(workspace.getCustomerId()) - .email(TEST_EMAIL) - .name(workspace.getName()) - .slug(workspace.getSlug()) - .initialSetupComplete(workspace.getInitialSetupComplete()) - .displaySetupWizard(workspace.getDisplaySetupWizard()) - .news(workspace.getNews()) - .anonymousDataCollection(workspace.getAnonymousDataCollection()) - .securityUpdates(workspace.getSecurityUpdates()) - .notifications(NotificationConverter.toApiList(workspace.getNotifications())) - .notificationSettings(NotificationSettingsConverter.toApi(workspace.getNotificationSettings())) - .defaultGeography(GEOGRAPHY_AUTO) - .organizationId(ORGANIZATION_ID); + final WorkspaceRead workspaceRead = getWorkspaceReadPerWorkspace(workspace); assertEquals(workspaceRead, workspacesHandler.getWorkspaceBySlug(slugRequestBody)); } @@ -543,7 +536,13 @@ void testGetWorkspaceByConnectionId() throws ConfigNotFoundException { final UUID connectionId = UUID.randomUUID(); when(configRepository.getStandardWorkspaceFromConnection(connectionId, false)).thenReturn(workspace); final ConnectionIdRequestBody connectionIdRequestBody = new ConnectionIdRequestBody().connectionId(connectionId); - final WorkspaceRead workspaceRead = new WorkspaceRead() + final WorkspaceRead workspaceRead = getWorkspaceReadPerWorkspace(workspace); + + assertEquals(workspaceRead, workspacesHandler.getWorkspaceByConnectionId(connectionIdRequestBody, false)); + } + + private WorkspaceRead getWorkspaceReadPerWorkspace(StandardWorkspace workspace) { + return new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) .customerId(workspace.getCustomerId()) .email(TEST_EMAIL) @@ -557,9 +556,8 @@ void testGetWorkspaceByConnectionId() throws ConfigNotFoundException { .notifications(NotificationConverter.toApiList(workspace.getNotifications())) .notificationSettings(NotificationSettingsConverter.toApi(workspace.getNotificationSettings())) .defaultGeography(GEOGRAPHY_AUTO) - .organizationId(ORGANIZATION_ID); - - assertEquals(workspaceRead, workspacesHandler.getWorkspaceByConnectionId(connectionIdRequestBody)); + .organizationId(ORGANIZATION_ID) + .tombstone(workspace.getTombstone()); } @Test @@ -568,7 +566,7 @@ void testGetWorkspaceByConnectionIdOnConfigNotFound() throws ConfigNotFoundExcep when(configRepository.getStandardWorkspaceFromConnection(connectionId, false)) .thenThrow(new ConfigNotFoundException("something", connectionId.toString())); final ConnectionIdRequestBody connectionIdRequestBody = new ConnectionIdRequestBody().connectionId(connectionId); - assertThrows(ConfigNotFoundException.class, () -> workspacesHandler.getWorkspaceByConnectionId(connectionIdRequestBody)); + assertThrows(ConfigNotFoundException.class, () -> workspacesHandler.getWorkspaceByConnectionId(connectionIdRequestBody, false)); } @ParameterizedTest @@ -660,7 +658,8 @@ void testUpdateWorkspace() .notificationSettings(generateApiNotificationSettings()) .defaultGeography(GEOGRAPHY_US) .webhookConfigs(List.of(new WebhookConfigRead().name(TEST_NAME).id(WEBHOOK_CONFIG_ID))) - .organizationId(ORGANIZATION_ID); + .organizationId(ORGANIZATION_ID) + .tombstone(false); final StandardWorkspace expectedWorkspaceWithSecrets = new StandardWorkspace() .withWorkspaceId(workspace.getWorkspaceId()) @@ -678,7 +677,8 @@ void testUpdateWorkspace() .withNotificationSettings(generateNotificationSettings()) .withDefaultGeography(Geography.US) .withWebhookOperationConfigs(SECRET_WEBHOOK_CONFIGS) - .withOrganizationId(ORGANIZATION_ID); + .withOrganizationId(ORGANIZATION_ID) + .withTombstone(false); verify(workspaceService).writeWorkspaceWithSecrets(expectedWorkspaceWithSecrets); @@ -766,7 +766,8 @@ void testUpdateWorkspaceNoNameUpdate() throws JsonValidationException, ConfigNot .notifications(List.of(generateApiNotification())) .notificationSettings(generateApiNotificationSettings()) .defaultGeography(GEOGRAPHY_AUTO) - .organizationId(ORGANIZATION_ID); + .organizationId(ORGANIZATION_ID) + .tombstone(false); verify(configRepository).writeStandardWorkspaceNoSecrets(expectedWorkspace); @@ -801,7 +802,8 @@ void testWorkspaceUpdateOrganization() .notifications(NotificationConverter.toApiList(workspace.getNotifications())) .notificationSettings(NotificationSettingsConverter.toApi(workspace.getNotificationSettings())) .defaultGeography(GEOGRAPHY_AUTO) - .organizationId(newOrgId); + .organizationId(newOrgId) + .tombstone(false); final WorkspaceRead actualWorkspaceRead = workspacesHandler.updateWorkspaceOrganization(workspaceUpdateOrganization); verify(workspaceService).writeStandardWorkspaceNoSecrets(expectedWorkspace); @@ -836,7 +838,8 @@ void testWorkspacePatchUpdate() throws JsonValidationException, ConfigNotFoundEx .notifications(NotificationConverter.toApiList(workspace.getNotifications())) .notificationSettings(NotificationSettingsConverter.toApi(workspace.getNotificationSettings())) .defaultGeography(GEOGRAPHY_AUTO) - .organizationId(ORGANIZATION_ID); + .organizationId(ORGANIZATION_ID) + .tombstone(false); final WorkspaceRead actualWorkspaceRead = workspacesHandler.updateWorkspace(workspaceUpdate); verify(configRepository).writeStandardWorkspaceNoSecrets(expectedWorkspace); @@ -890,7 +893,8 @@ void testWorkspaceIsWrittenThroughSecretsWriter() .notifications(List.of(generateApiNotification())) .notificationSettings(generateApiNotificationSettingsWithDefaultValue()) .defaultGeography(GEOGRAPHY_US) - .webhookConfigs(Collections.emptyList()); + .webhookConfigs(Collections.emptyList()) + .tombstone(false); assertEquals(expectedRead, actualRead); verify(workspaceService, times(1)).writeWorkspaceWithSecrets(any()); diff --git a/airbyte-commons-temporal-core/src/main/java/io/airbyte/commons/temporal/utils/PayloadChecker.java b/airbyte-commons-temporal-core/src/main/java/io/airbyte/commons/temporal/utils/PayloadChecker.java index 50eca140307..3c9a8505507 100644 --- a/airbyte-commons-temporal-core/src/main/java/io/airbyte/commons/temporal/utils/PayloadChecker.java +++ b/airbyte-commons-temporal-core/src/main/java/io/airbyte/commons/temporal/utils/PayloadChecker.java @@ -7,6 +7,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.exception.SizeLimitException; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.OssMetricsRegistry; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -23,10 +25,16 @@ */ public class PayloadChecker { + private MetricClient metricClient; + private static final Logger log = LoggerFactory.getLogger(PayloadChecker.class); public static final int MAX_PAYLOAD_SIZE_BYTES = 4 * 1024 * 1024; + public PayloadChecker(final MetricClient metricClient) { + this.metricClient = metricClient; + } + /** * Validate the payload size fits within temporal message size limits. * @@ -35,16 +43,17 @@ public class PayloadChecker { * @return data if size is valid * @throws SizeLimitException if payload size exceeds temporal limits. */ - public static T validatePayloadSize(final T data) { + public T validatePayloadSize(final T data) { final String serializedData = Jsons.serialize(data); if (serializedData.length() > MAX_PAYLOAD_SIZE_BYTES) { emitInspectionLog(data); + metricClient.count(OssMetricsRegistry.PAYLOAD_SIZE_EXCEEDED, 1); throw new SizeLimitException(String.format("Complete result exceeds size limit (%s of %s)", serializedData.length(), MAX_PAYLOAD_SIZE_BYTES)); } return data; } - private static void emitInspectionLog(final T data) { + private void emitInspectionLog(final T data) { final JsonNode jsonData = Jsons.jsonNode(data); final Map inspectionMap = new HashMap<>(); for (Iterator it = jsonData.fieldNames(); it.hasNext();) { diff --git a/airbyte-commons-temporal-core/src/test/java/io/airbyte/commons/temporal/utils/PayloadCheckerTest.java b/airbyte-commons-temporal-core/src/test/java/io/airbyte/commons/temporal/utils/PayloadCheckerTest.java index e35fad370a7..17a9adc768e 100644 --- a/airbyte-commons-temporal-core/src/test/java/io/airbyte/commons/temporal/utils/PayloadCheckerTest.java +++ b/airbyte-commons-temporal-core/src/test/java/io/airbyte/commons/temporal/utils/PayloadCheckerTest.java @@ -6,24 +6,30 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; import io.airbyte.commons.temporal.exception.SizeLimitException; +import io.airbyte.metrics.lib.MetricClient; import org.junit.jupiter.api.Test; class PayloadCheckerTest { + MetricClient mMetricClient = mock(MetricClient.class); + + PayloadChecker payloadChecker = new PayloadChecker(mMetricClient); + record Payload(String data) {} @Test void testValidPayloadSize() { final Payload p = new Payload("1".repeat(PayloadChecker.MAX_PAYLOAD_SIZE_BYTES - "{\"data\":\"\"}".length())); - assertEquals(p, PayloadChecker.validatePayloadSize(p)); + assertEquals(p, payloadChecker.validatePayloadSize(p)); } @Test void testInvalidPayloadSize() { final Payload p = new Payload("1".repeat(PayloadChecker.MAX_PAYLOAD_SIZE_BYTES)); - assertThrows(SizeLimitException.class, () -> PayloadChecker.validatePayloadSize(p)); + assertThrows(SizeLimitException.class, () -> payloadChecker.validatePayloadSize(p)); } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DestinationTimeoutMonitor.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DestinationTimeoutMonitor.java index fb115d25364..1d064aa07d4 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DestinationTimeoutMonitor.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/DestinationTimeoutMonitor.java @@ -280,12 +280,12 @@ public static class TimeoutException extends RuntimeException { public final String humanReadableThreshold; public final String humanReadableTimeSinceLastAction; - public TimeoutException(final long threshold, final long timeSinceLastAction) { + public TimeoutException(final long thresholdMs, final long timeSinceLastActionMs) { super(String.format("Last action %s ago, exceeding the threshold of %s.", - DurationFormatUtils.formatDurationWords(timeSinceLastAction, true, true), - DurationFormatUtils.formatDurationWords(threshold, true, true))); - this.humanReadableThreshold = DurationFormatUtils.formatDurationWords(threshold, true, true); - this.humanReadableTimeSinceLastAction = DurationFormatUtils.formatDurationWords(threshold, true, true); + DurationFormatUtils.formatDurationWords(timeSinceLastActionMs, true, true), + DurationFormatUtils.formatDurationWords(thresholdMs, true, true))); + this.humanReadableThreshold = DurationFormatUtils.formatDurationWords(thresholdMs, true, true); + this.humanReadableTimeSinceLastAction = DurationFormatUtils.formatDurationWords(timeSinceLastActionMs, true, true); } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java index 1bb36342c9f..b205c234cb5 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java @@ -100,7 +100,8 @@ public HeartbeatTimeoutChaperone(final HeartbeatMonitor heartbeatMonitor, * @throws ExecutionException - throw is the runnable throw an exception */ public void runWithHeartbeatThread(final CompletableFuture runnableFuture) throws ExecutionException { - LOGGER.info("Starting source heartbeat check. Will check every {} minutes.", timeoutCheckDuration.toMinutes()); + LOGGER.info("Starting source heartbeat check. Will check threshold of {} seconds, every {} minutes.", + heartbeatMonitor.getHeartbeatFreshnessThreshold().toSeconds(), timeoutCheckDuration.toMinutes()); final CompletableFuture heartbeatFuture = CompletableFuture.runAsync(customMonitor.orElse(this::monitor), getLazyExecutorService()); try { @@ -127,9 +128,9 @@ public void runWithHeartbeatThread(final CompletableFuture runnableFuture) new MetricAttribute(MetricTags.CONNECTION_ID, connectionId.toString()), new MetricAttribute(MetricTags.KILLED, "true"), new MetricAttribute(MetricTags.SOURCE_IMAGE, sourceDockerImage)); - final var threshold = heartbeatMonitor.getHeartbeatFreshnessThreshold().getSeconds(); - final var timeBetweenLastRecord = heartbeatMonitor.getTimeSinceLastBeat().orElse(Duration.ZERO).getSeconds(); - throw new HeartbeatTimeoutException(threshold, timeBetweenLastRecord); + final var thresholdMs = heartbeatMonitor.getHeartbeatFreshnessThreshold().toMillis(); + final var timeBetweenLastRecordMs = heartbeatMonitor.getTimeSinceLastBeat().orElse(Duration.ZERO).toMillis(); + throw new HeartbeatTimeoutException(thresholdMs, timeBetweenLastRecordMs); } else { LOGGER.info("Do not terminate as feature flag is disable"); metricClient.count(OssMetricsRegistry.SOURCE_HEARTBEAT_FAILURE, 1, @@ -188,12 +189,12 @@ public static class HeartbeatTimeoutException extends RuntimeException { public final String humanReadableThreshold; public final String humanReadableTimeSinceLastRec; - public HeartbeatTimeoutException(final long threshold, final long timeBetweenLastRecord) { - super(String.format("Last record saw %s ago, exceeding the threshold of %s.", - DurationFormatUtils.formatDurationWords(timeBetweenLastRecord, true, true), - DurationFormatUtils.formatDurationWords(threshold, true, true))); - this.humanReadableThreshold = DurationFormatUtils.formatDurationWords(threshold, true, true); - this.humanReadableTimeSinceLastRec = DurationFormatUtils.formatDurationWords(threshold, true, true); + public HeartbeatTimeoutException(final long thresholdMs, final long timeBetweenLastRecordMs) { + super(String.format("Last record seen %s ago, exceeding the threshold of %s.", + DurationFormatUtils.formatDurationWords(timeBetweenLastRecordMs, true, true), + DurationFormatUtils.formatDurationWords(thresholdMs, true, true))); + this.humanReadableThreshold = DurationFormatUtils.formatDurationWords(thresholdMs, true, true); + this.humanReadableTimeSinceLastRec = DurationFormatUtils.formatDurationWords(timeBetweenLastRecordMs, true, true); } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java index 3ccd0ae1864..8296412c464 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactory.java @@ -78,7 +78,7 @@ public record InvalidLineFailureConfiguration(boolean failTooLongRecords, boolea private static final int BUFFER_READ_AHEAD_LIMIT = 2 * 1024 * 1024; // 2 megabytes private static final int MESSAGES_LOOK_AHEAD_FOR_DETECTION = 10; private static final String TYPE_FIELD_NAME = "type"; - private static final int MAXIMUM_CHARACTERS_ALLOWED = 5_000_000; + private static final int MAXIMUM_CHARACTERS_ALLOWED = 20_000_000; // BASIC PROCESSING FIELDS protected final Logger logger; @@ -366,6 +366,8 @@ private String humanReadableByteCountSI(long bytes) { * 3. upgrade the message to the platform version, if needed. */ protected Stream toAirbyteMessage(final String line) { + handleCannotDeserialize(line); + Optional m = deserializer.deserializeExact(line); if (m.isPresent()) { @@ -379,7 +381,6 @@ protected Stream toAirbyteMessage(final String line) { return upgradeMessage(m.get()); } - handleCannotDeserialize(line); return m.stream(); } @@ -400,13 +401,15 @@ protected Stream toAirbyteMessage(final String line) { private void handleCannotDeserialize(final String line) { try (final MdcScope ignored = containerLogMdcBuilder.build()) { if (line.length() >= MAXIMUM_CHARACTERS_ALLOWED) { - MetricClientFactory.getMetricClient().count(OssMetricsRegistry.LINE_SKIPPED_TOO_LONG, 1); + connectionId.ifPresentOrElse(c -> MetricClientFactory.getMetricClient().count(OssMetricsRegistry.LINE_SKIPPED_TOO_LONG, 1, + new MetricAttribute(MetricTags.CONNECTION_ID, c.toString())), + () -> MetricClientFactory.getMetricClient().count(OssMetricsRegistry.LINE_SKIPPED_TOO_LONG, 1)); MetricClientFactory.getMetricClient().distribution(OssMetricsRegistry.TOO_LONG_LINES_DISTRIBUTION, line.length()); if (invalidLineFailureConfiguration.printLongRecordPks) { - LOGGER.error("[LARGE RECORD] A record is too long with size: " + line.length()); + LOGGER.warn("[LARGE RECORD] A record is too long with size: " + line.length()); configuredAirbyteCatalog.ifPresent( airbyteCatalog -> LOGGER - .error("[LARGE RECORD] The primary keys of the long record are: " + gsonPksExtractor.extractPks(airbyteCatalog, line))); + .warn("[LARGE RECORD] The primary keys of the long record are: " + gsonPksExtractor.extractPks(airbyteCatalog, line))); } if (invalidLineFailureConfiguration.failTooLongRecords) { if (exceptionClass.isPresent()) { @@ -421,12 +424,16 @@ private void handleCannotDeserialize(final String line) { // Connectors can sometimes log error messages from failing to parse an AirbyteRecordMessage. // Filter on record into debug to try and prevent such cases. Though this catches non-record // messages, this is ok as we rather be safe than sorry. + logger.warn("Could not parse the string received from source, it seems to be a record message"); connectionId.ifPresentOrElse(c -> MetricClientFactory.getMetricClient().count(OssMetricsRegistry.LINE_SKIPPED_WITH_RECORD, 1, new MetricAttribute(MetricTags.CONNECTION_ID, c.toString())), () -> MetricClientFactory.getMetricClient().count(OssMetricsRegistry.LINE_SKIPPED_WITH_RECORD, 1)); logger.debug(line); } else { - MetricClientFactory.getMetricClient().count(OssMetricsRegistry.NON_AIRBYTE_MESSAGE_LOG_LINE, 1); + connectionId.ifPresentOrElse( + c -> MetricClientFactory.getMetricClient().count(OssMetricsRegistry.NON_AIRBYTE_MESSAGE_LOG_LINE, 1, + new MetricAttribute(MetricTags.CONNECTION_ID, c.toString())), + () -> MetricClientFactory.getMetricClient().count(OssMetricsRegistry.NON_AIRBYTE_MESSAGE_LOG_LINE, 1)); logger.info(line); } } catch (final Exception e) { diff --git a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/general/ReplicationWorkerHelper.kt b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/general/ReplicationWorkerHelper.kt index e40ffff59d0..df641e5226b 100644 --- a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/general/ReplicationWorkerHelper.kt +++ b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/general/ReplicationWorkerHelper.kt @@ -59,7 +59,6 @@ import io.airbyte.workload.api.client.model.generated.WorkloadHeartbeatRequest import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.http.HttpStatus import org.apache.commons.io.FileUtils -import org.openapitools.client.infrastructure.ClientException import org.slf4j.MDC import java.nio.file.Path import java.time.Duration @@ -67,6 +66,7 @@ import java.time.Instant import java.util.Collections import java.util.Optional import java.util.concurrent.atomic.AtomicBoolean +import io.airbyte.workload.api.client.generated.infrastructure.ClientException as GeneratedClientException private val logger = KotlinLogging.logger { } @@ -146,13 +146,15 @@ class ReplicationWorkerHelper( * Workload should stop because it is no longer expected to be running. * See [io.airbyte.workload.api.WorkloadApi.workloadHeartbeat] */ - if (e is ClientException && e.statusCode == HttpStatus.GONE.code) { - logger.warn(e) { "Received kill response from API, shutting down heartbeat" } + if (e is GeneratedClientException && e.statusCode == HttpStatus.GONE.code) { + metricClient.count(OssMetricsRegistry.HEARTBEAT_TERMINAL_SHUTDOWN, 1, *metricAttrs.toTypedArray()) markCancelled() return@Runnable } else if (Duration.between(lastSuccessfulHeartbeat, Instant.now()) > heartbeatTimeoutDuration) { logger.warn(e) { "Have not been able to update heartbeat for more than the timeout duration, shutting down heartbeat" } + metricClient.count(OssMetricsRegistry.HEARTBEAT_CONNECTIVITY_FAILURE_SHUTDOWN, 1, *metricAttrs.toTypedArray()) markFailed() + abort() trackFailure(WorkloadHeartbeatException("Workload Heartbeat Error", e)) return@Runnable } @@ -450,8 +452,7 @@ class ReplicationWorkerHelper( val failures = mutableListOf() // only .setFailures() if a failure occurred or if there is an AirbyteErrorTraceMessage - messageTracker.errorTraceMessageFailure(context.jobId, context.attempt) - ?.let { failures.add(it) } + failures.addAll(messageTracker.errorTraceMessageFailure(context.jobId, context.attempt)) failures.addAll(replicationFailures) diff --git a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/AnalyticsMessageTracker.kt b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/AnalyticsMessageTracker.kt index a256b9ed5a0..f48abba9746 100644 --- a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/AnalyticsMessageTracker.kt +++ b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/AnalyticsMessageTracker.kt @@ -45,7 +45,7 @@ class AnalyticsMessageTracker(private val trackingClient: TrackingClient) { } } - private fun generateAnalyticsMetadata(currentMessages: List): Map? { + private fun generateAnalyticsMetadata(currentMessages: List): Map { val context = requireNotNull(ctx) val jsonList: ArrayNode = Jsons.arrayNode() jsonList.addAll(currentMessages) diff --git a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/AirbyteMessageTracker.kt b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/AirbyteMessageTracker.kt index de60459c143..d55ef490de5 100644 --- a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/AirbyteMessageTracker.kt +++ b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/AirbyteMessageTracker.kt @@ -10,6 +10,7 @@ import io.airbyte.workers.helper.FailureHelper import io.airbyte.workers.internal.stateaggregator.DefaultStateAggregator import io.airbyte.workers.internal.stateaggregator.StateAggregator import io.github.oshai.kotlinlogging.KotlinLogging +import java.util.ArrayList private val logger = KotlinLogging.logger {} @@ -72,17 +73,12 @@ class AirbyteMessageTracker( fun errorTraceMessageFailure( jobId: Long, attempt: Int, - ): FailureReason? { - val srcMsg = srcErrorTraceMsgs.firstOrNull() - val dstMsg = dstErrorTraceMsgs.firstOrNull() - - return when { - srcMsg == null && dstMsg == null -> null - srcMsg != null && dstMsg == null -> FailureHelper.sourceFailure(srcMsg, jobId, attempt) - srcMsg == null && dstMsg != null -> FailureHelper.destinationFailure(dstMsg, jobId, attempt) - srcMsg != null && dstMsg != null && srcMsg.emittedAt <= dstMsg.emittedAt -> FailureHelper.sourceFailure(srcMsg, jobId, attempt) - else -> FailureHelper.destinationFailure(dstMsg, jobId, attempt) - } + ): List { + val allErrors = + srcErrorTraceMsgs.map { + FailureHelper.sourceFailure(it, jobId, attempt) + } + dstErrorTraceMsgs.map { FailureHelper.destinationFailure(it, jobId, attempt) } + return allErrors.sortedBy { it.getTimestamp() } } /** diff --git a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/ParallelStreamStatsTracker.kt b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/ParallelStreamStatsTracker.kt index e1500b36efb..76570f05aa9 100644 --- a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/ParallelStreamStatsTracker.kt +++ b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/ParallelStreamStatsTracker.kt @@ -118,7 +118,11 @@ class ParallelStreamStatsTracker( else -> { val statsTracker = getOrCreateStreamStatsTracker(getNameNamespacePair(stateMessage)) statsTracker.trackStateFromSource(stateMessage) - updateChecksumValidationStatus(statsTracker.areStreamStatsReliable(), AirbyteMessageOrigin.SOURCE, getNameNamespacePair(stateMessage)) + updateChecksumValidationStatus( + statsTracker.areStreamStatsReliable(), + AirbyteMessageOrigin.SOURCE, + getNameNamespacePair(stateMessage), + ) validateStateChecksum( stateMessage, statsTracker.getTrackedEmittedRecordsSinceLastStateMessage().toDouble(), @@ -307,15 +311,18 @@ class ParallelStreamStatsTracker( if (!shouldEmitStateStatsToSegment(stateMessage)) { return } - val payload: MutableMap = HashMap() - payload["connection_id"] = connectionId.toString() - payload["job_id"] = jobId.toString() - payload["attempt_number"] = attemptNumber.toString() - payload["state_origin"] = stateOrigin - payload["record_count"] = recordCount.toString() - payload["valid_data"] = checksumValidationEnabled.toString() - payload["state_type"] = stateMessage.type.toString() - payload["state_hash"] = stateMessage.getStateHashCode(Hashing.murmur3_32_fixed()).toString() + val payload: MutableMap = + mutableMapOf( + "connection_id" to connectionId.toString(), + "job_id" to jobId.toString(), + "attempt_number" to attemptNumber.toString(), + "state_origin" to stateOrigin, + "record_count" to recordCount.toString(), + "valid_data" to checksumValidationEnabled.toString(), + "state_type" to stateMessage.type.toString(), + "state_hash" to stateMessage.getStateHashCode(Hashing.murmur3_32_fixed()).toString(), + ) + if (stateMessage.type == AirbyteStateMessage.AirbyteStateType.STREAM) { val nameNamespacePair = getNameNamespacePair(stateMessage) if (nameNamespacePair.namespace != null) { diff --git a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/StatsTracker.kt b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/StatsTracker.kt index be8aba4fd1f..60577b8342e 100644 --- a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/StatsTracker.kt +++ b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/internal/bookkeeping/StatsTracker.kt @@ -224,13 +224,13 @@ class StreamStatsTracker( return } - logger.info { "Id of the state message received from the destination $stateId" } + logger.debug { "Id of the state message received from the destination $stateId" } var stagedStats: StagedStats? = null // un-stage stats until the stateMessage while (!stagedStatsList.isEmpty()) { stagedStats = stagedStatsList.poll() - logger.info { + logger.debug { "removing ${stagedStats.stateId} from the stored stateIds for the stream " + "${nameNamespacePair.namespace}:${nameNamespacePair.name}, " + "state received time ${stagedStats.receivedTime}" + diff --git a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/storage/StorageClient.kt b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/storage/StorageClient.kt index 01563bb2b82..451ae7e5321 100644 --- a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/storage/StorageClient.kt +++ b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/storage/StorageClient.kt @@ -299,6 +299,7 @@ internal fun GcsStorageConfig.gcsClient(): Storage { */ internal fun MinioStorageConfig.s3Client(): S3Client = S3Client.builder() + .serviceConfiguration { it.pathStyleAccessEnabled(true) } .credentialsProvider { AwsBasicCredentials.create(this@s3Client.accessKey, this@s3Client.secretAccessKey) } .endpointOverride(URI(this@s3Client.endpoint)) // The region isn't actually used but is required. Set to us-east-1 based on https://github.com/minio/minio/discussions/15063. diff --git a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/sync/OrchestratorConstants.kt b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/sync/OrchestratorConstants.kt index e4b8faf317b..21dc35f47c0 100644 --- a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/sync/OrchestratorConstants.kt +++ b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/sync/OrchestratorConstants.kt @@ -80,6 +80,7 @@ object OrchestratorConstants { EnvVar.AWS_SECRET_ACCESS_KEY, EnvVar.DD_AGENT_HOST, EnvVar.DD_DOGSTATSD_PORT, + EnvVar.DOCKER_HOST, EnvVar.GOOGLE_APPLICATION_CREDENTIALS, EnvVar.JOB_DEFAULT_ENV_MAP, EnvVar.JOB_ISOLATED_KUBE_NODE_SELECTORS, diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java index 5d8b8a392af..2f0b31ababf 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java @@ -605,7 +605,7 @@ void testReplicationRunnableDestinationFailure() throws Exception { @Test void testReplicationRunnableDestinationFailureViaTraceMessage() throws Exception { final FailureReason failureReason = FailureHelper.destinationFailure(ERROR_TRACE_MESSAGE, Long.valueOf(JOB_ID), JOB_ATTEMPT); - when(messageTracker.errorTraceMessageFailure(Long.parseLong(JOB_ID), JOB_ATTEMPT)).thenReturn(failureReason); + when(messageTracker.errorTraceMessageFailure(Long.parseLong(JOB_ID), JOB_ATTEMPT)).thenReturn(List.of(failureReason)); final ReplicationWorker worker = getDefaultReplicationWorker(); @@ -1137,12 +1137,21 @@ void testGetFailureReason() { assertEquals(failureReason.getFailureOrigin(), FailureOrigin.SOURCE); failureReason = ReplicationWorkerHelper.getFailureReason(new DestinationException(""), jobId, attempt); assertEquals(failureReason.getFailureOrigin(), FailureOrigin.DESTINATION); - failureReason = ReplicationWorkerHelper.getFailureReason(new HeartbeatTimeoutChaperone.HeartbeatTimeoutException(10, 15), jobId, attempt); + failureReason = ReplicationWorkerHelper.getFailureReason(new HeartbeatTimeoutChaperone.HeartbeatTimeoutException(10000, 15000), jobId, attempt); assertEquals(failureReason.getFailureOrigin(), FailureOrigin.SOURCE); assertEquals(failureReason.getFailureType(), FailureReason.FailureType.HEARTBEAT_TIMEOUT); + assertEquals( + "Airbyte detected that the Source didn't send any records in the last 15 seconds, exceeding the configured 10 seconds threshold. Airbyte will try reading again on the next sync. Please see https://docs.airbyte.com/understanding-airbyte/heartbeats for more info.", + failureReason.getExternalMessage()); + assertEquals("Last record seen 15 seconds ago, exceeding the threshold of 10 seconds.", failureReason.getInternalMessage()); failureReason = ReplicationWorkerHelper.getFailureReason(new RuntimeException(), jobId, attempt); assertEquals(failureReason.getFailureOrigin(), FailureOrigin.REPLICATION); - failureReason = ReplicationWorkerHelper.getFailureReason(new TimeoutException(10, 15), jobId, attempt); + failureReason = ReplicationWorkerHelper.getFailureReason(new TimeoutException(10000, 15000), jobId, attempt); + assertEquals( + "Airbyte detected that the Destination didn't make progress in the last 15 seconds, exceeding the configured 10 seconds threshold. Airbyte will try reading again on the next sync. Please see https://docs.airbyte.com/understanding-airbyte/heartbeats for more info.", + failureReason.getExternalMessage()); + assertEquals("Last action 15 seconds ago, exceeding the threshold of 10 seconds.", failureReason.getInternalMessage()); + System.out.println(failureReason.getInternalMessage()); assertEquals(failureReason.getFailureOrigin(), FailureOrigin.DESTINATION); assertEquals(failureReason.getFailureType(), FailureType.DESTINATION_TIMEOUT); } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java index bee9a0f49e9..242e1359e70 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java @@ -72,6 +72,14 @@ void testGenericFailureFromTrace() throws Exception { assertEquals(FailureType.CONFIG_ERROR, failureReason.getFailureType()); } + @Test + void testFailureWithTransientFailureType() { + final AirbyteTraceMessage traceMessage = + AirbyteMessageUtils.createErrorTraceMessage("sample trace message", 10.0, AirbyteErrorTraceMessage.FailureType.TRANSIENT_ERROR); + final FailureReason reason = FailureHelper.genericFailure(traceMessage, 1034L, 0); + assertEquals(FailureType.TRANSIENT_ERROR, reason.getFailureType()); + } + @Test void testGenericFailureFromTraceNoFailureType() throws Exception { final FailureReason failureReason = FailureHelper.genericFailure(TRACE_MESSAGE, Long.valueOf(12345), 1); diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/HeartBeatTimeoutChaperoneTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/HeartBeatTimeoutChaperoneTest.java index 275cc023fa0..3ae658273bc 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/HeartBeatTimeoutChaperoneTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/HeartBeatTimeoutChaperoneTest.java @@ -5,6 +5,8 @@ package io.airbyte.workers.internal; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -24,7 +26,6 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; class HeartBeatTimeoutChaperoneTest { @@ -40,6 +41,8 @@ class HeartBeatTimeoutChaperoneTest { @Test void testFailHeartbeat() { when(featureFlagClient.boolVariation(eq(ShouldFailSyncIfHeartbeatFailure.INSTANCE), any())).thenReturn(true); + when(heartbeatMonitor.getHeartbeatFreshnessThreshold()).thenReturn(Duration.ofSeconds(1)); + final HeartbeatTimeoutChaperone heartbeatTimeoutChaperone = new HeartbeatTimeoutChaperone( heartbeatMonitor, timeoutCheckDuration, @@ -49,14 +52,17 @@ void testFailHeartbeat() { connectionId, metricClient); - Assertions.assertThatThrownBy(() -> heartbeatTimeoutChaperone.runWithHeartbeatThread(CompletableFuture.runAsync(() -> { - try { - Thread.sleep(Long.MAX_VALUE); - } catch (final InterruptedException e) { - throw new RuntimeException(e); - } - }))) - .isInstanceOf(HeartbeatTimeoutChaperone.HeartbeatTimeoutException.class); + final var thrown = assertThrows(HeartbeatTimeoutChaperone.HeartbeatTimeoutException.class, + () -> heartbeatTimeoutChaperone.runWithHeartbeatThread(CompletableFuture.runAsync(() -> { + try { + Thread.sleep(Long.MAX_VALUE); + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + }))); + + assertEquals("Last record seen 0 seconds ago, exceeding the threshold of 1 second.", thrown.getMessage()); + verify(metricClient, times(1)).count(OssMetricsRegistry.SOURCE_HEARTBEAT_FAILURE, 1, new MetricAttribute(MetricTags.CONNECTION_ID, connectionId.toString()), new MetricAttribute(MetricTags.KILLED, "true"), diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactoryTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactoryTest.java index aab2b98fa08..e74e21cb698 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactoryTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactoryTest.java @@ -241,7 +241,7 @@ void testToAirbyteMessageVeryLongMessageDontFail() { longStringBuilder.append("a"); } final String messageLine = String.format(VALID_MESSAGE_TEMPLATE, longStringBuilder); - Assertions.assertThat(getFactory(false).toAirbyteMessage(messageLine)).isEmpty(); + Assertions.assertThat(getFactory(false).toAirbyteMessage(messageLine)).isNotEmpty(); } private Stream stringToMessageStream(final String inputString) { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/bookkeeping/AirbyteMessageTrackerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/bookkeeping/AirbyteMessageTrackerTest.java index 44529479f0b..742293ae029 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/bookkeeping/AirbyteMessageTrackerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/bookkeeping/AirbyteMessageTrackerTest.java @@ -5,7 +5,7 @@ package io.airbyte.workers.internal.bookkeeping; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -18,6 +18,9 @@ import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.workers.helper.FailureHelper; import io.airbyte.workers.test_utils.AirbyteMessageUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -193,8 +196,12 @@ void testErrorTraceMessageFailureWithMultipleTraceErrors() { messageTracker.acceptFromDestination(dstMsg1); messageTracker.acceptFromDestination(dstMsg2); - final FailureReason failureReason = FailureHelper.sourceFailure(srcMsg1.getTrace(), Long.valueOf(123), 1); - assertEquals(messageTracker.errorTraceMessageFailure(123L, 1), failureReason); + List failureReasons = new ArrayList<>(); + failureReasons.addAll( + Stream.of(srcMsg1, srcMsg2).map(m -> FailureHelper.sourceFailure(m.getTrace(), Long.valueOf(123), 1)).toList()); + failureReasons.addAll( + Stream.of(dstMsg1, dstMsg2).map(m -> FailureHelper.destinationFailure(m.getTrace(), Long.valueOf(123), 1)).toList()); + assertEquals(messageTracker.errorTraceMessageFailure(123L, 1), failureReasons); } @Test @@ -203,12 +210,12 @@ void testErrorTraceMessageFailureWithOneTraceError() { messageTracker.acceptFromDestination(destMessage); final FailureReason failureReason = FailureHelper.destinationFailure(destMessage.getTrace(), Long.valueOf(123), 1); - assertEquals(messageTracker.errorTraceMessageFailure(123L, 1), failureReason); + assertEquals(messageTracker.errorTraceMessageFailure(123L, 1), List.of(failureReason)); } @Test void testErrorTraceMessageFailureWithNoTraceErrors() { - assertNull(messageTracker.errorTraceMessageFailure(123L, 1)); + assertTrue(messageTracker.errorTraceMessageFailure(123L, 1).isEmpty()); } } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 0171620958f..1e7b141116b 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -8,6 +8,7 @@ import static java.util.stream.Collectors.toMap; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.core.util.Separators; @@ -45,8 +46,18 @@ @SuppressWarnings({"PMD.AvoidReassigningParameters", "PMD.AvoidCatchingThrowable"}) public class Jsons { + private static final StreamReadConstraints STREAM_READ_CONSTRAINTS = StreamReadConstraints + .builder() + .maxStringLength(Integer.MAX_VALUE) + .build(); + // Object Mapper is thread-safe private static final ObjectMapper OBJECT_MAPPER = MoreMappers.initMapper(); + + static { + OBJECT_MAPPER.getFactory().setStreamReadConstraints(STREAM_READ_CONSTRAINTS); + } + /** * Exact ObjectMapper preserves float information by using the Java Big Decimal type. */ @@ -55,6 +66,7 @@ public class Jsons { static { OBJECT_MAPPER_EXACT = MoreMappers.initMapper(); OBJECT_MAPPER_EXACT.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + OBJECT_MAPPER_EXACT.getFactory().setStreamReadConstraints(STREAM_READ_CONSTRAINTS); } private static final ObjectWriter OBJECT_WRITER = OBJECT_MAPPER.writer(new JsonPrettyPrinter()); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java index d11418ac725..5039154dc47 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java @@ -11,9 +11,11 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.yaml.Yamls; import java.nio.charset.Charset; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.Logger; @@ -42,11 +44,18 @@ public class MaskedDataInterceptor implements RewritePolicy { protected static final Logger logger = StatusLogger.getLogger(); + // This is a little circuitous, but it gets the regex syntax highlighting in intelliJ to work. + private static final String DESTINATION_ERROR_PREFIX = Pattern.compile("^(?.*destination.*\\s+>\\s+ERROR.+)").pattern(); + /** * The pattern used to determine if a message contains sensitive data. */ private final Optional pattern; + private static final List KNOWN_PII_PATTERNS = List.of( + Pattern.compile(DESTINATION_ERROR_PREFIX + "(?Received\\s+invalid\\s+message:)(.+)$"), + Pattern.compile(DESTINATION_ERROR_PREFIX + "(?org\\.jooq\\.exception\\.DataAccessException: SQL.+values\\s+\\()(.+)$")); + @PluginFactory public static MaskedDataInterceptor createPolicy( @PluginAttribute(value = "specMaskFile", @@ -82,11 +91,21 @@ public LogEvent rewrite(final LogEvent source) { * @return The possibly masked log message. */ private String applyMask(final String message) { - if (pattern.isPresent()) { - return message.replaceAll(pattern.get(), "\"$1\":\"" + AirbyteSecretConstants.SECRETS_MASK + "\""); - } else { - return message; - } + final String piiScrubbedMessage = removeKnownPii(message); + return pattern.map(s -> piiScrubbedMessage.replaceAll(s, "\"$1\":\"" + AirbyteSecretConstants.SECRETS_MASK + "\"")) + .orElse(piiScrubbedMessage); + } + + /** + * Removes known PII from the message. + * + * @param message the log line + * @return a redacted log line + */ + private static String removeKnownPii(final String message) { + return KNOWN_PII_PATTERNS.stream() + .reduce(message, (msg, pattern) -> pattern.matcher(msg).replaceAll( + "${destinationPrefix}${messagePrefix}" + AirbyteSecretConstants.SECRETS_MASK), (a, b) -> a); } /** diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java index f6a22b06908..78f1590d2d4 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java @@ -19,27 +19,6 @@ public AirbyteVersion(final String major, final String minor, final String patch super(major, minor, patch); } - /** - * Test if versions are compatible. Only the major and minor part of the Version is taken into - * account. - * - * @param version1 to test - * @param version2 to test - * @throws IllegalStateException if they are not compatible - */ - public static void assertIsCompatible(final AirbyteVersion version1, final AirbyteVersion version2) throws IllegalStateException { - if (!isCompatible(version1, version2)) { - throw new IllegalStateException(getErrorMessage(version1, version2)); - } - } - - private static String getErrorMessage(final AirbyteVersion version1, final AirbyteVersion version2) { - return String.format( - "Version mismatch between %s and %s.\n" - + "Please upgrade or reset your Airbyte Database, see more at https://docs.airbyte.io/operator-guides/upgrading-airbyte", - version1.serialize(), version2.serialize()); - } - @Override public String toString() { return "AirbyteVersion{" @@ -50,29 +29,4 @@ public String toString() { + '}'; } - /** - * Convert a version to itself without its patch version. - * - * @param airbyteVersion to convert - * @return version without patch - */ - public static AirbyteVersion versionWithoutPatch(final AirbyteVersion airbyteVersion) { - final String versionWithoutPatch = "" + airbyteVersion.getMajorVersion() - + "." - + airbyteVersion.getMinorVersion() - + ".0-" - + airbyteVersion.serialize().replace("\n", "").strip().split("-")[1]; - return new AirbyteVersion(versionWithoutPatch); - } - - /** - * Convert a string representation of a version to itself without its patch version. - * - * @param airbyteVersion to convert - * @return version without patch - */ - public static AirbyteVersion versionWithoutPatch(final String airbyteVersion) { - return versionWithoutPatch(new AirbyteVersion(airbyteVersion)); - } - } diff --git a/airbyte-commons/src/main/kotlin/io/airbyte/commons/envvar/EnvVar.kt b/airbyte-commons/src/main/kotlin/io/airbyte/commons/envvar/EnvVar.kt index a4af450ae98..dc476f03daf 100644 --- a/airbyte-commons/src/main/kotlin/io/airbyte/commons/envvar/EnvVar.kt +++ b/airbyte-commons/src/main/kotlin/io/airbyte/commons/envvar/EnvVar.kt @@ -31,6 +31,7 @@ enum class EnvVar { DD_VERSION, DEPLOYMENT_ENV, DEPLOYMENT_MODE, + DOCKER_HOST, DOCKER_NETWORK, FEATURE_FLAG_CLIENT, diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/MaskedDataInterceptorTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/MaskedDataInterceptorTest.java index 23f4d566c1c..af81de9f6c5 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/logging/MaskedDataInterceptorTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/logging/MaskedDataInterceptorTest.java @@ -32,6 +32,28 @@ class MaskedDataInterceptorTest { private static final String JSON_WITHOUT_SECRETS = "{\"prop1\":\"test\",\"" + OTHER + "\":{\"prop2\":\"value\",\"prop3\":1234}}"; public static final String TEST_SPEC_SECRET_MASK_YAML = "/test_spec_secret_mask.yaml"; + public static final String TEST_LOGGED_RECORD_CONTENTS = + "2024-03-21 12:19:08 \u001B[43mdestination\u001B[0m > ERROR i.a.c.i.b.Destination$ShimToSerializedAirbyteMessageConsumer(consumeMessage):120 " + + "Received invalid message: {\"type\":\"RECORD\",\"record\":{\"namespace\":\""; + public static final String REDACTED_LOGGED_RECORD_CONTENTS = + "2024-03-21 12:19:08 \u001B[43mdestination\u001B[0m > ERROR i.a.c.i.b.Destination$ShimToSerializedAirbyteMessageConsumer(consumeMessage):120 " + + "Received invalid message:" + + AirbyteSecretConstants.SECRETS_MASK; + public static final String TEST_LOGGED_SQL_VALUES = + "2024-03-19 20:03:43 \u001B[43mdestination\u001B[0m > ERROR pool-4-thread-1 i.a.c.i.d.a.FlushWorkers(flush$lambda$6):192 Flush Worker (632c9) " + + "-- flush worker " + + "error: java.lang.RuntimeException: org.jooq.exception.DataAccessException: SQL [insert into " + + "\"airbyte_internal\".\"public_raw__stream_foo\" (_airbyte_raw_id, _airbyte_data, _airbyte_meta, _airbyte_extracted_at, " + + "_airbyte_loaded_at) values ('UUID', a bunch of other stuff"; + + public static final String REDACTED_LOGGED_SQL_VALUES = + "2024-03-19 20:03:43 \u001B[43mdestination\u001B[0m > ERROR pool-4-thread-1 i.a.c.i.d.a.FlushWorkers(flush$lambda$6):192 Flush Worker (632c9) " + + "-- flush worker " + + "error: java.lang.RuntimeException: org.jooq.exception.DataAccessException: SQL [insert into " + + "\"airbyte_internal\".\"public_raw__stream_foo\" (_airbyte_raw_id, _airbyte_data, _airbyte_meta, _airbyte_extracted_at, " + + "_airbyte_loaded_at) values (" + + AirbyteSecretConstants.SECRETS_MASK; + @Test void testMaskingMessageWithStringSecret() { final Message message = mock(Message.class); @@ -125,4 +147,30 @@ void testMissingMaskingFileDoesNotPreventLogging() { }); } + @Test + void testMaskingMessageWithSqlValues() { + final Message message = mock(Message.class); + final LogEvent logEvent = mock(LogEvent.class); + when(message.getFormattedMessage()).thenReturn(TEST_LOGGED_SQL_VALUES); + when(logEvent.getMessage()).thenReturn(message); + + final MaskedDataInterceptor interceptor = MaskedDataInterceptor.createPolicy(TEST_SPEC_SECRET_MASK_YAML); + + final LogEvent result = interceptor.rewrite(logEvent); + assertEquals(REDACTED_LOGGED_SQL_VALUES, result.getMessage().getFormattedMessage()); + } + + @Test + void testMaskingMessageWithRecordContents() { + final Message message = mock(Message.class); + final LogEvent logEvent = mock(LogEvent.class); + when(message.getFormattedMessage()).thenReturn(TEST_LOGGED_RECORD_CONTENTS); + when(logEvent.getMessage()).thenReturn(message); + + final MaskedDataInterceptor interceptor = MaskedDataInterceptor.createPolicy(TEST_SPEC_SECRET_MASK_YAML); + + final LogEvent result = interceptor.rewrite(logEvent); + assertEquals(REDACTED_LOGGED_RECORD_CONTENTS, result.getMessage().getFormattedMessage()); + } + } diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java index 571c4e01eb7..b48e34163b2 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java @@ -105,12 +105,6 @@ void testSerialize() { assertEquals(nonDevVersion, new AirbyteVersion(nonDevVersion).serialize()); } - @Test - void testCheckVersion() { - AirbyteVersion.assertIsCompatible(new AirbyteVersion("3.2.1"), new AirbyteVersion("3.2.1")); - assertThrows(IllegalStateException.class, () -> AirbyteVersion.assertIsCompatible(new AirbyteVersion("1.2.3"), new AirbyteVersion("3.2.1"))); - } - @Test void testCheckOnlyPatchVersion() { assertFalse(new AirbyteVersion(VERSION_678).checkOnlyPatchVersionIsUpdatedComparedTo(new AirbyteVersion(VERSION_678))); diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/storage/MinioS3ClientFactory.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/storage/MinioS3ClientFactory.java deleted file mode 100644 index b6956feb76f..00000000000 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/storage/MinioS3ClientFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.storage; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.function.Supplier; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; - -/** - * When using minio, we can still leverage the S3Client, we just slightly change what information we - * pass to it. Takes in the constructor our standard format for minio configuration and provides a - * factory that uses that configuration to create an S3Client. - */ -@SuppressWarnings("PMD.AvoidThrowingRawExceptionTypes") -public class MinioS3ClientFactory implements Supplier { - - private final MinioStorageConfig config; - - public MinioS3ClientFactory(final MinioStorageConfig config) { - this.config = config; - } - - @Override - public S3Client get() { - final var builder = S3Client.builder(); - - // The Minio S3 client. - final var minioEndpoint = config.getEndpoint(); - try { - final var minioUri = new URI(minioEndpoint); - builder.credentialsProvider(() -> AwsBasicCredentials.create(config.getAccessKey(), config.getSecretAccessKey())); - builder.endpointOverride(minioUri); - builder.region(Region.US_EAST_1); // Although this is not used, the S3 client will error out if this is not set. Set a stub value. - } catch (final URISyntaxException e) { - throw new RuntimeException("Error creating S3 log client to Minio", e); - } - - return builder.build(); - } - -} diff --git a/airbyte-config/config-models/src/main/kotlin/io/airbyte/config/storage/MinioS3ClientFactory.kt b/airbyte-config/config-models/src/main/kotlin/io/airbyte/config/storage/MinioS3ClientFactory.kt new file mode 100644 index 00000000000..d5f192f2a94 --- /dev/null +++ b/airbyte-config/config-models/src/main/kotlin/io/airbyte/config/storage/MinioS3ClientFactory.kt @@ -0,0 +1,32 @@ +package io.airbyte.config.storage + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.S3Client +import java.net.URI +import java.util.function.Supplier + +class MinioS3ClientFactory(private val config: MinioStorageConfig) : Supplier { + override fun get(): S3Client = + runCatching { + val minioUri = URI(config.endpoint) + + with(S3Client.builder()) { + serviceConfiguration { + it.pathStyleAccessEnabled(true) + } + credentialsProvider { + AwsBasicCredentials.create( + config.accessKey, + config.secretAccessKey, + ) + } + endpointOverride(minioUri) + // Although this is not used, the S3 client will error out if this is not set. + region(Region.US_EAST_1) + build() + } + }.getOrElse { + throw RuntimeException("Error creating S3 log client to Minio", it) + } +} diff --git a/airbyte-config/config-models/src/main/resources/types/FailureReason.yaml b/airbyte-config/config-models/src/main/resources/types/FailureReason.yaml index e99cf18954e..839b8b994ce 100644 --- a/airbyte-config/config-models/src/main/resources/types/FailureReason.yaml +++ b/airbyte-config/config-models/src/main/resources/types/FailureReason.yaml @@ -29,6 +29,7 @@ properties: - refresh_schema - heartbeat_timeout - destination_timeout + - transient_error internalMessage: description: Human readable failure description for consumption by technical system operators, like Airbyte engineers or OSS users. type: string diff --git a/airbyte-config/config-models/src/main/resources/types/InvitationStatus.yaml b/airbyte-config/config-models/src/main/resources/types/InvitationStatus.yaml index 6f0fdcab0f3..0c13f937544 100644 --- a/airbyte-config/config-models/src/main/resources/types/InvitationStatus.yaml +++ b/airbyte-config/config-models/src/main/resources/types/InvitationStatus.yaml @@ -9,3 +9,4 @@ enum: - accepted - cancelled - declined + - expired diff --git a/airbyte-config/config-models/src/main/resources/types/UserInvitation.yaml b/airbyte-config/config-models/src/main/resources/types/UserInvitation.yaml index 3ea0ea6dc08..851beb72d4d 100644 --- a/airbyte-config/config-models/src/main/resources/types/UserInvitation.yaml +++ b/airbyte-config/config-models/src/main/resources/types/UserInvitation.yaml @@ -13,6 +13,7 @@ required: - scopeType - permissionType - status + - expiresAt additionalProperties: true properties: id: @@ -29,6 +30,10 @@ properties: description: Email address of the user who is being invited type: string format: email + acceptedByUserId: + description: ID of the user who accepted the invitation + type: string + format: uuid scopeId: description: ID of the workspace/organization that the user is being invited to type: string @@ -50,3 +55,7 @@ properties: description: last updated timestamp of the invitation type: integer format: int64 + expiresAt: + description: Timestamp at which the invitation will expire + type: integer + format: int64 diff --git a/airbyte-config/config-models/src/test/java/io/airbyte/config/storage/MinioS3ClientFactoryTest.java b/airbyte-config/config-models/src/test/java/io/airbyte/config/storage/MinioS3ClientFactoryTest.java deleted file mode 100644 index 0900899872d..00000000000 --- a/airbyte-config/config-models/src/test/java/io/airbyte/config/storage/MinioS3ClientFactoryTest.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.storage; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; - -import org.junit.jupiter.api.Test; - -class MinioS3ClientFactoryTest { - - @Test - void testMinio() { - final var bucket = new StorageBucketConfig("log", "state", "workload"); - final var config = new MinioStorageConfig(bucket, "access", "secret", "http://endpoint.test"); - - assertDoesNotThrow(() -> new MinioS3ClientFactory(config).get()); - } - -} diff --git a/airbyte-config/config-models/src/test/kotlin/io/airbyte/config/storage/MinioS3ClientFactoryTest.kt b/airbyte-config/config-models/src/test/kotlin/io/airbyte/config/storage/MinioS3ClientFactoryTest.kt new file mode 100644 index 00000000000..5808545fe8b --- /dev/null +++ b/airbyte-config/config-models/src/test/kotlin/io/airbyte/config/storage/MinioS3ClientFactoryTest.kt @@ -0,0 +1,14 @@ +package io.airbyte.config.storage + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertDoesNotThrow + +val buckets = StorageBucketConfig(log = "log", state = "state", workloadOutput = "workload") +val config = MinioStorageConfig(buckets = buckets, accessKey = "access", secretAccessKey = "secret", endpoint = "http://endpoint.test") + +class MinioS3ClientFactoryTest { + @Test + fun `minio doesn't throw exception`() { + assertDoesNotThrow { MinioS3ClientFactory(config).get() } + } +} diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigNotFoundException.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigNotFoundException.java index 06d96d10d9e..d178f932ca9 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigNotFoundException.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigNotFoundException.java @@ -12,6 +12,11 @@ */ public class ConfigNotFoundException extends Exception { + // This is a specific error type that is used when an organization cannot be found + // from a given workspace. Workspaces will soon require an organization, so this + // error is temporary and will be removed once the requirement is enforced. + public static final String NO_ORGANIZATION_FOR_WORKSPACE = "NO_ORGANIZATION_FOR_WORKSPACE"; + private static final long serialVersionUID = 836273627; private final String type; private final String configId; diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java index 2382add7e01..77125ad9dd3 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java @@ -278,6 +278,19 @@ public Optional getUserByEmail(final String email) throws IOException { return Optional.of(createUserFromRecord(result.get(0))); } + /** + * Fetch all users with a given email address. + */ + public List getUsersByEmail(final String email) throws IOException { + return database.query(ctx -> ctx + .select(asterisk()) + .from(USER) + .where(USER.EMAIL.eq(email)).fetch()) + .stream() + .map(this::createUserFromRecord) + .toList(); + } + /** * Get the default user if it exists by looking up the hardcoded default user id. */ diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/version_overrides/ConfigurationDefinitionVersionOverrideProvider.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/version_overrides/ConfigurationDefinitionVersionOverrideProvider.java index cf730d59d7e..7f963928e0f 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/version_overrides/ConfigurationDefinitionVersionOverrideProvider.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/version_overrides/ConfigurationDefinitionVersionOverrideProvider.java @@ -17,6 +17,13 @@ import io.airbyte.data.services.ScopedConfigurationService; import io.airbyte.data.services.WorkspaceService; import io.airbyte.data.services.shared.ConnectorVersionKey; +import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.featureflag.UseActorScopedDefaultVersions; +import io.airbyte.featureflag.UseBreakingChangeScopedConfigs; +import io.airbyte.featureflag.Workspace; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.validation.json.JsonValidationException; import jakarta.inject.Named; import jakarta.inject.Singleton; @@ -34,13 +41,19 @@ public class ConfigurationDefinitionVersionOverrideProvider implements Definitio private final WorkspaceService workspaceService; private final ActorDefinitionService actorDefinitionService; private final ScopedConfigurationService scopedConfigurationService; + private final FeatureFlagClient featureFlagClient; + private final MetricClient metricClient; public ConfigurationDefinitionVersionOverrideProvider(final WorkspaceService workspaceService, final ActorDefinitionService actorDefinitionService, - final ScopedConfigurationService scopedConfigurationService) { + final ScopedConfigurationService scopedConfigurationService, + final FeatureFlagClient featureFlagClient, + final MetricClient metricClient) { this.workspaceService = workspaceService; this.actorDefinitionService = actorDefinitionService; this.scopedConfigurationService = scopedConfigurationService; + this.featureFlagClient = featureFlagClient; + this.metricClient = metricClient; } private UUID getOrganizationId(final UUID workspaceId) { @@ -82,6 +95,24 @@ public Optional getOverride(final Acto final Optional optConfig = getScopedConfig(actorDefinitionId, workspaceId, actorId); if (optConfig.isPresent()) { final ScopedConfiguration config = optConfig.get(); + if (config.getOriginType() == ConfigOriginType.BREAKING_CHANGE) { + if (featureFlagClient.boolVariation(UseActorScopedDefaultVersions.INSTANCE, new Workspace(workspaceId))) { + // If the above feature flag is off, defaultVersion won't consider breaking change setbacks, + // so metrics wouldn't be accurate + final String status = defaultVersion.getVersionId().toString().equals(config.getValue()) ? "ok" : "invalid"; + metricClient.count(OssMetricsRegistry.CONNECTOR_BREAKING_CHANGE_PIN_SERVED, 1, + new MetricAttribute("workspace_id", workspaceId.toString()), + new MetricAttribute("actor_id", actorId != null ? actorId.toString() : "null"), + new MetricAttribute("actor_default_version", defaultVersion.getVersionId().toString()), + new MetricAttribute("pinned_version", config.getValue()), + new MetricAttribute("status", status)); + } + + if (!featureFlagClient.boolVariation(UseBreakingChangeScopedConfigs.INSTANCE, new Workspace(workspaceId))) { + return Optional.empty(); + } + } + try { final ActorDefinitionVersion version = actorDefinitionService.getActorDefinitionVersion(UUID.fromString(config.getValue())); final boolean isManualOverride = config.getOriginType() == ConfigOriginType.USER; diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java index eff9117aab8..dc485719c41 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java @@ -148,6 +148,16 @@ void getUserByEmailTest() throws IOException { } } + @Test + void getUsersByEmailTest() throws IOException { + for (final User user : MockData.dupEmailUsers()) { + userPersistence.writeUser(user); + } + + final List usersWithSameEmail = userPersistence.getUsersByEmail(MockData.DUP_EMAIL); + Assertions.assertEquals(new HashSet<>(MockData.dupEmailUsers()), new HashSet<>(usersWithSameEmail)); + } + @Test void deleteUserByIdTest() throws IOException { userPersistence.deleteUserById(MockData.CREATOR_USER_ID_1); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/version_overrides/ConfigurationDefinitionVersionOverrideProviderTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/version_overrides/ConfigurationDefinitionVersionOverrideProviderTest.java index 818b7bc2e02..ade11d17e79 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/version_overrides/ConfigurationDefinitionVersionOverrideProviderTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/version_overrides/ConfigurationDefinitionVersionOverrideProviderTest.java @@ -7,6 +7,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -32,6 +34,14 @@ import io.airbyte.data.services.ScopedConfigurationService; import io.airbyte.data.services.WorkspaceService; import io.airbyte.data.services.shared.ConnectorVersionKey; +import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.featureflag.TestClient; +import io.airbyte.featureflag.UseActorScopedDefaultVersions; +import io.airbyte.featureflag.UseBreakingChangeScopedConfigs; +import io.airbyte.featureflag.Workspace; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -69,6 +79,7 @@ class ConfigurationDefinitionVersionOverrideProviderTest { .withNormalizationTag("tag") .withNormalizationIntegrationType("bigquery"); private static final ActorDefinitionVersion DEFAULT_VERSION = new ActorDefinitionVersion() + .withVersionId(UUID.randomUUID()) .withDockerRepository(DOCKER_REPOSITORY) .withActorDefinitionId(ACTOR_DEFINITION_ID) .withDockerImageTag(DOCKER_IMAGE_TAG) @@ -81,6 +92,7 @@ class ConfigurationDefinitionVersionOverrideProviderTest { .withSupportsDbt(true) .withNormalizationConfig(NORMALIZATION_CONFIG); private static final ActorDefinitionVersion OVERRIDE_VERSION = new ActorDefinitionVersion() + .withVersionId(UUID.randomUUID()) .withDockerRepository(DOCKER_REPOSITORY) .withActorDefinitionId(ACTOR_DEFINITION_ID) .withDockerImageTag(DOCKER_IMAGE_TAG_2) @@ -96,6 +108,8 @@ class ConfigurationDefinitionVersionOverrideProviderTest { private WorkspaceService mWorkspaceService; private ActorDefinitionService mActorDefinitionService; private ScopedConfigurationService mScopedConfigurationService; + private FeatureFlagClient mFeatureFlagClient; + private MetricClient mMetricClient; private ConfigurationDefinitionVersionOverrideProvider overrideProvider; @BeforeEach @@ -103,10 +117,19 @@ void setup() throws JsonValidationException, ConfigNotFoundException, IOExceptio mWorkspaceService = mock(WorkspaceService.class); mActorDefinitionService = mock(ActorDefinitionService.class); mScopedConfigurationService = mock(ScopedConfigurationService.class); - overrideProvider = new ConfigurationDefinitionVersionOverrideProvider(mWorkspaceService, mActorDefinitionService, mScopedConfigurationService); + mFeatureFlagClient = mock(TestClient.class); + mMetricClient = mock(MetricClient.class); + overrideProvider = new ConfigurationDefinitionVersionOverrideProvider(mWorkspaceService, mActorDefinitionService, mScopedConfigurationService, + mFeatureFlagClient, mMetricClient); when(mWorkspaceService.getStandardWorkspaceNoSecrets(WORKSPACE_ID, true)) .thenReturn(new StandardWorkspace().withOrganizationId(ORGANIZATION_ID)); + + when(mFeatureFlagClient.boolVariation(eq(UseActorScopedDefaultVersions.INSTANCE), any())) + .thenReturn(true); + + when(mFeatureFlagClient.boolVariation(eq(UseBreakingChangeScopedConfigs.INSTANCE), any())) + .thenReturn(true); } @Test @@ -243,4 +266,75 @@ void testThrowsIfVersionIdDoesNotExist() throws ConfigNotFoundException, IOExcep verifyNoMoreInteractions(mScopedConfigurationService, mActorDefinitionService); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testBCPinIntegrityMetricsEmitted(final Boolean withMismatchedVersions) throws ConfigNotFoundException, IOException { + final UUID versionId = OVERRIDE_VERSION.getVersionId(); + final ScopedConfiguration breakingChangePin = new ScopedConfiguration() + .withId(UUID.randomUUID()) + .withScopeType(ConfigScopeType.WORKSPACE) + .withScopeId(WORKSPACE_ID) + .withResourceType(ConfigResourceType.ACTOR_DEFINITION) + .withResourceId(ACTOR_DEFINITION_ID) + .withValue(versionId.toString()) + .withOriginType(ConfigOriginType.BREAKING_CHANGE); + + when(mScopedConfigurationService.getScopedConfiguration(ConnectorVersionKey.INSTANCE, ConfigResourceType.ACTOR_DEFINITION, ACTOR_DEFINITION_ID, + Map.of( + ConfigScopeType.ORGANIZATION, ORGANIZATION_ID, + ConfigScopeType.WORKSPACE, WORKSPACE_ID, + ConfigScopeType.ACTOR, ACTOR_ID))) + .thenReturn(Optional.of(breakingChangePin)); + + when(mActorDefinitionService.getActorDefinitionVersion(versionId)).thenReturn(OVERRIDE_VERSION); + + final ActorDefinitionVersion defaultVersion = withMismatchedVersions ? DEFAULT_VERSION : OVERRIDE_VERSION; + + final Optional optResult = + overrideProvider.getOverride(ActorType.SOURCE, ACTOR_DEFINITION_ID, WORKSPACE_ID, ACTOR_ID, defaultVersion); + + assertTrue(optResult.isPresent()); + assertEquals(OVERRIDE_VERSION, optResult.get().actorDefinitionVersion()); + + verify(mMetricClient).count(OssMetricsRegistry.CONNECTOR_BREAKING_CHANGE_PIN_SERVED, 1, + new MetricAttribute("workspace_id", WORKSPACE_ID.toString()), + new MetricAttribute("actor_id", ACTOR_ID.toString()), + new MetricAttribute("actor_default_version", defaultVersion.getVersionId().toString()), + new MetricAttribute("pinned_version", versionId.toString()), + new MetricAttribute("status", withMismatchedVersions ? "invalid" : "ok")); + } + + @Test + void testBCPinIntegrityMetricsNotEmittedWhenFFOff() throws ConfigNotFoundException, IOException { + when(mFeatureFlagClient.boolVariation(UseActorScopedDefaultVersions.INSTANCE, new Workspace(WORKSPACE_ID))) + .thenReturn(false); + + final UUID versionId = OVERRIDE_VERSION.getVersionId(); + final ScopedConfiguration breakingChangePin = new ScopedConfiguration() + .withId(UUID.randomUUID()) + .withScopeType(ConfigScopeType.WORKSPACE) + .withScopeId(WORKSPACE_ID) + .withResourceType(ConfigResourceType.ACTOR_DEFINITION) + .withResourceId(ACTOR_DEFINITION_ID) + .withValue(versionId.toString()) + .withOriginType(ConfigOriginType.BREAKING_CHANGE); + + when(mScopedConfigurationService.getScopedConfiguration(ConnectorVersionKey.INSTANCE, ConfigResourceType.ACTOR_DEFINITION, ACTOR_DEFINITION_ID, + Map.of( + ConfigScopeType.ORGANIZATION, ORGANIZATION_ID, + ConfigScopeType.WORKSPACE, WORKSPACE_ID, + ConfigScopeType.ACTOR, ACTOR_ID))) + .thenReturn(Optional.of(breakingChangePin)); + + when(mActorDefinitionService.getActorDefinitionVersion(versionId)).thenReturn(OVERRIDE_VERSION); + + final Optional optResult = + overrideProvider.getOverride(ActorType.SOURCE, ACTOR_DEFINITION_ID, WORKSPACE_ID, ACTOR_ID, DEFAULT_VERSION); + + assertTrue(optResult.isPresent()); + assertEquals(OVERRIDE_VERSION, optResult.get().actorDefinitionVersion()); + + verifyNoInteractions(mMetricClient); + } + } diff --git a/airbyte-config/config-persistence/src/testFixtures/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/config-persistence/src/testFixtures/java/io/airbyte/config/persistence/MockData.java index fa8a7beb162..127ebf1d8af 100644 --- a/airbyte-config/config-persistence/src/testFixtures/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/config-persistence/src/testFixtures/java/io/airbyte/config/persistence/MockData.java @@ -132,6 +132,9 @@ public class MockData { static final UUID CREATOR_USER_ID_3 = UUID.randomUUID(); static final UUID CREATOR_USER_ID_4 = UUID.randomUUID(); static final UUID CREATOR_USER_ID_5 = UUID.randomUUID(); + static final UUID DUP_EMAIL_USER_ID_1 = UUID.randomUUID(); + static final UUID DUP_EMAIL_USER_ID_2 = UUID.randomUUID(); + static final String DUP_EMAIL = "dup-email@airbyte.io"; // Permission static final UUID PERMISSION_ID_1 = UUID.randomUUID(); @@ -295,6 +298,34 @@ public static List users() { return Arrays.asList(user1, user2, user3, user4, user5); } + public static List dupEmailUsers() { + final User dupEmailUser1 = new User() + .withUserId(DUP_EMAIL_USER_ID_1) + .withName("dup-email-user-1") + .withAuthUserId(DUP_EMAIL_USER_ID_1.toString()) + .withAuthProvider(AuthProvider.KEYCLOAK) + .withDefaultWorkspaceId(null) + .withStatus(User.Status.REGISTERED) + .withCompanyName("dup-user-company") + .withEmail(DUP_EMAIL) + .withNews(true) + .withUiMetadata(null); + + final User dupEmailUser2 = new User() + .withUserId(DUP_EMAIL_USER_ID_2) + .withName("dup-email-user-2") + .withAuthUserId(DUP_EMAIL_USER_ID_2.toString()) + .withAuthProvider(AuthProvider.KEYCLOAK) + .withDefaultWorkspaceId(null) + .withStatus(User.Status.REGISTERED) + .withCompanyName("dup-user-company") + .withEmail(DUP_EMAIL) + .withNews(true) + .withUiMetadata(null); + + return Arrays.asList(dupEmailUser1, dupEmailUser2); + } + public static List permissions() { return Arrays.asList(permission1, permission2, permission3, permission4, permission5, permission6, permission7); } diff --git a/airbyte-config/init/Dockerfile b/airbyte-config/init/Dockerfile index b18390e8d59..fd9cd7d20eb 100644 --- a/airbyte-config/init/Dockerfile +++ b/airbyte-config/init/Dockerfile @@ -1,4 +1,4 @@ -ARG ALPINE_IMAGE=alpine:3.13 +ARG ALPINE_IMAGE=alpine:3.18 FROM ${ALPINE_IMAGE} AS seed WORKDIR /app diff --git a/airbyte-config/init/src/main/resources/icons/propel.svg b/airbyte-config/init/src/main/resources/icons/propel.svg new file mode 100644 index 00000000000..3c8d80a317d --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/propel.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/icons/zenhub.svg b/airbyte-config/init/src/main/resources/icons/zenhub.svg new file mode 100644 index 00000000000..ccca65a2b49 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/zenhub.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-connector-builder-resources/CDK_VERSION b/airbyte-connector-builder-resources/CDK_VERSION index afed694eede..31b648bd6fa 100644 --- a/airbyte-connector-builder-resources/CDK_VERSION +++ b/airbyte-connector-builder-resources/CDK_VERSION @@ -1 +1 @@ -0.65.0 +0.73.0 diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index ef0d3631df6..9d10aa41394 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -1,8 +1,8 @@ -ARG BASE_IMAGE=airbyte/airbyte-base-java-python-image:1.1.0 -FROM ${BASE_IMAGE} AS connector-builder-server +ARG JAVA_PYTHON_BASE_IMAGE_VERSION=2.1.0 +FROM airbyte/airbyte-base-java-python-image:${JAVA_PYTHON_BASE_IMAGE_VERSION} AS connector-builder-server # Set up CDK requirements -ARG CDK_VERSION=0.65.0 +ARG CDK_VERSION=0.73.0 ENV CDK_PYTHON=${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin/python ENV CDK_ENTRYPOINT ${PYENV_ROOT}/versions/${PYTHON_VERSION}/lib/python3.9/site-packages/airbyte_cdk/connector_builder/main.py # Set up CDK @@ -18,7 +18,10 @@ ENV VERSION ${VERSION} WORKDIR /app # This is automatically unzipped by Docker +USER root ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"] diff --git a/airbyte-connector-builder-server/README.md b/airbyte-connector-builder-server/README.md index 3e69357b68c..714cfa73d38 100644 --- a/airbyte-connector-builder-server/README.md +++ b/airbyte-connector-builder-server/README.md @@ -22,25 +22,67 @@ export CDK_PYTHON= export CDK_ENTRYPOINT= ``` +Example commands: +``` +export CDK_PYTHON=~/code/airbyte/airbyte-cdk/python/.venv/bin/python +export CDK_ENTRYPOINT=~/code/airbyte/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py +``` + Then run the server (You can also do this w/o build) ```bash ./gradlew -p oss airbyte-connector-builder-server:run ``` -The server is now reachable on localhost:80 +The server is now reachable on localhost:8080 If you want to run the full platform with this local instance, you must edit the `.env` file as follows: ``` bash # replace this -CONNECTOR_BUILDER_SERVER_API_HOST=http://airbyte-connector-builder-server:80 +CONNECTOR_BUILDER_SERVER_API_HOST=http://airbyte-connector-builder-server:8080 # with this -CONNECTOR_BUILDER_SERVER_API_HOST=http://host.docker.internal:80 +CONNECTOR_BUILDER_SERVER_API_HOST=http://host.docker.internal:8080 ``` Note: there are two different, but very similarly-named, environment variables; you must edit `CONNECTOR_BUILDER_SERVER_API_HOST`, not `CONNECTOR_BUILDER_API_HOST`. +### Running the platform with support for custom components (docker-compose only) + +1. Run the OSS platform locally with builder docker-compose extension + 1. Example command: PATH_TO_CONNECTORS=/Users/alex/code/airbyte/airbyte-integrations/connectors docker compose -f docker-compose.yaml -f docker-compose.builder.yaml up + 2. Where PATH_TO_CONNECTORS points to the airbyte-integrations/connectors subdirectory in the opensource airbyte repository +2. Open the connector builder and develop your connector +3. When needing a custom componentt: + 1. Switch to the YAML view + 2. Define the custom component +4. Write the custom components and its unit tests +5. Run test read + +Note that connector modules are added to the path at startup time. The platform instance must be restarted if you add a new connector module. + +Follow these additional instructions if the connector requires 3rd party libraries that are not available in the CDK: + +Developing connectors that require 3rd party libraries can be done by running the connector-builder-server locally and pointing to a custom virtual environment. + +1. Create a virtual environment and install the CDK + any 3rd party library required +2. export CDK_PYTHON= + - `CDK_PYTHON` should point to the virtual environment's python executable (example: `export CDK_PYTHON=~/code/airbyte/airbyte-cdk/python/.venv/bin/python`) +3. export CDK_ENTRYPOINT= +4. ./gradlew -p oss airbyte-connector-builder-server:run + 1. The server is now reachable on localhost:8080 +5. Update the server to point to port 8080 by editing .env and replacing + + ``` + CONNECTOR_BUILDER_SERVER_API_HOST=http://airbyte-connector-builder-server:8080 + ``` + with + ``` + CONNECTOR_BUILDER_SERVER_API_HOST=http://host.docker.internal:8080 + ``` + +6. Follow the standard instructions + ## OpenAPI generation Run it via Gradle by running this from the Airbyte project root: diff --git a/airbyte-connector-builder-server/requirements.in b/airbyte-connector-builder-server/requirements.in index 7b51c89f5aa..818351f449e 100644 --- a/airbyte-connector-builder-server/requirements.in +++ b/airbyte-connector-builder-server/requirements.in @@ -1 +1 @@ -airbyte-cdk==0.65.0 +airbyte-cdk==0.73.0 diff --git a/airbyte-connector-builder-server/requirements.txt b/airbyte-connector-builder-server/requirements.txt index b3d69874c7e..b65f0497dca 100644 --- a/airbyte-connector-builder-server/requirements.txt +++ b/airbyte-connector-builder-server/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile # -airbyte-cdk==0.65.0 +airbyte-cdk==0.73.0 # via -r requirements.in airbyte-protocol-models==0.5.1 # via airbyte-cdk @@ -17,7 +17,7 @@ backoff==2.2.1 # via airbyte-cdk bracex==2.4 # via wcmatch -cachetools==5.3.2 +cachetools==5.3.3 # via airbyte-cdk cattrs==23.2.3 # via requests-cache @@ -57,7 +57,7 @@ pyrate-limiter==3.1.1 # via airbyte-cdk pyrsistent==0.20.0 # via jsonschema -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # airbyte-cdk # pendulum @@ -77,7 +77,7 @@ six==1.16.0 # jsonschema # python-dateutil # url-normalize -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via # cattrs # pydantic diff --git a/airbyte-connector-builder-server/src/main/resources/application.yml b/airbyte-connector-builder-server/src/main/resources/application.yml index daa4d6fb7b0..651bab096ca 100644 --- a/airbyte-connector-builder-server/src/main/resources/application.yml +++ b/airbyte-connector-builder-server/src/main/resources/application.yml @@ -25,7 +25,7 @@ micronaut: authentication-provider-strategy: ALL enabled: ${API_AUTHORIZATION_ENABLED:false} server: - port: 80 + port: 8080 cors: enabled: true netty: diff --git a/airbyte-connector-sidecar/Dockerfile b/airbyte-connector-sidecar/Dockerfile index 58466c6ba08..0f0a284d136 100644 --- a/airbyte-connector-sidecar/Dockerfile +++ b/airbyte-connector-sidecar/Dockerfile @@ -1,4 +1,5 @@ -FROM amazoncorretto:21 AS connector-sidecar +ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.1.0 +FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION} ARG DOCKER_BUILD_ARCH=amd64 @@ -8,12 +9,13 @@ ARG VERSION=dev ENV APPLICATION airbyte-connector-sidecar ENV VERSION=${VERSION} -WORKDIR /app - +USER root COPY WellKnownTypes.json /app # Move connector-sidecar app ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "/app/airbyte-app/bin/${APPLICATION}"] diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 33a71448a5a..765b6e9a765 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -1,4 +1,5 @@ -FROM airbyte/airbyte-base-java-worker-image:2.0.1 +ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.1.0 +FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION} # Don't change this manually. Bump version expects to make moves based on this string ARG VERSION=dev @@ -8,10 +9,13 @@ ENV VERSION=${VERSION} WORKDIR /app +USER root COPY WellKnownTypes.json /app # Move orchestrator app ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "/app/airbyte-app/bin/${APPLICATION}"] diff --git a/airbyte-container-orchestrator/src/main/resources/application-k8s.yml b/airbyte-container-orchestrator/src/main/resources/application-k8s.yml new file mode 100644 index 00000000000..2226fc3d3ce --- /dev/null +++ b/airbyte-container-orchestrator/src/main/resources/application-k8s.yml @@ -0,0 +1,9 @@ +micronaut: + caches: + # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for + # track events + # We overwrite the expiry in orchestrator to avoid making the workspace API call again and again for state stats metrics + analytics-tracking-deployments: + expire-after-access: 24h + analytics-tracking-identity: + expire-after-access: 24h diff --git a/airbyte-cron/Dockerfile b/airbyte-cron/Dockerfile index ee6007bedcb..b83e567150e 100644 --- a/airbyte-cron/Dockerfile +++ b/airbyte-cron/Dockerfile @@ -1,5 +1,11 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.0.1 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.1.0 FROM ${JDK_IMAGE} + WORKDIR /app + +USER root ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte + ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/airbyte-cron"] diff --git a/airbyte-cron/build.gradle.kts b/airbyte-cron/build.gradle.kts index 706f2d5c2e4..032063cb4ab 100644 --- a/airbyte-cron/build.gradle.kts +++ b/airbyte-cron/build.gradle.kts @@ -1,77 +1,80 @@ import java.util.Properties plugins { - id("io.airbyte.gradle.jvm.app") - id("io.airbyte.gradle.docker") - id("io.airbyte.gradle.publish") - kotlin("jvm") - kotlin("kapt") + id("io.airbyte.gradle.jvm.app") + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") + kotlin("jvm") + kotlin("kapt") } dependencies { - compileOnly(libs.lombok) - annotationProcessor(libs.lombok) // Lombok must be added BEFORE Micronaut - annotationProcessor(platform(libs.micronaut.platform)) - annotationProcessor(libs.bundles.micronaut.annotation.processor) - kapt(libs.bundles.micronaut.annotation.processor) + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) // Lombok must be added BEFORE Micronaut + annotationProcessor(platform(libs.micronaut.platform)) + annotationProcessor(libs.bundles.micronaut.annotation.processor) + kapt(libs.bundles.micronaut.annotation.processor) - implementation(platform(libs.micronaut.platform)) - implementation(libs.bundles.micronaut) - implementation(libs.bundles.micronaut.cache) - implementation(libs.bundles.micronaut.metrics) - implementation(libs.bundles.kubernetes.client) - implementation(libs.bundles.temporal) - implementation(libs.bundles.datadog) - implementation(libs.failsafe) - implementation(libs.failsafe.okhttp) - implementation(libs.java.jwt) - implementation(libs.kotlin.logging) - implementation(libs.okhttp) - implementation(libs.sentry.java) - implementation(libs.lombok) - implementation(libs.commons.io) + implementation(platform(libs.micronaut.platform)) + implementation(libs.bundles.micronaut) + implementation(libs.bundles.micronaut.cache) + implementation(libs.bundles.micronaut.metrics) + implementation(libs.bundles.kubernetes.client) + implementation(libs.bundles.temporal) + implementation(libs.bundles.datadog) + implementation(libs.failsafe) + implementation(libs.failsafe.okhttp) + implementation(libs.java.jwt) + implementation(libs.kotlin.logging) + implementation(libs.okhttp) + implementation(libs.sentry.java) + implementation(libs.lombok) + implementation(libs.commons.io) - implementation(project(":airbyte-api")) - implementation(project(":airbyte-analytics")) - implementation(project(":airbyte-commons")) - implementation(project(":airbyte-commons-auth")) - implementation(project(":airbyte-commons-micronaut")) - implementation(project(":airbyte-commons-temporal")) - implementation(project(":airbyte-config:config-models")) - implementation(project(":airbyte-config:config-persistence")) - implementation(project(":airbyte-config:init")) - implementation(project(":airbyte-json-validation")) - implementation(project(":airbyte-data")) - implementation(project(":airbyte-db:db-lib")) - implementation(project(":airbyte-featureflag")) - implementation(project(":airbyte-metrics:metrics-lib")) - implementation(project(":airbyte-persistence:job-persistence")) + implementation(project(":airbyte-api")) + implementation(project(":airbyte-analytics")) + implementation(project(":airbyte-commons")) + implementation(project(":airbyte-commons-auth")) + implementation(project(":airbyte-commons-micronaut")) + implementation(project(":airbyte-commons-temporal")) + implementation(project(":airbyte-config:config-models")) + implementation(project(":airbyte-config:config-persistence")) + implementation(project(":airbyte-config:init")) + implementation(project(":airbyte-json-validation")) + implementation(project(":airbyte-data")) + implementation(project(":airbyte-db:db-lib")) + implementation(project(":airbyte-featureflag")) + implementation(project(":airbyte-metrics:metrics-lib")) + implementation(project(":airbyte-persistence:job-persistence")) - runtimeOnly(libs.snakeyaml) + runtimeOnly(libs.snakeyaml) - testImplementation(libs.bundles.junit) - testImplementation(libs.mockk) + testImplementation(libs.bundles.junit) + testImplementation(libs.mockk) } -val env = Properties().apply { +val env = + Properties().apply { load(rootProject.file(".env.dev").inputStream()) -} + } airbyte { - application { - mainClass = "io.airbyte.cron.MicronautCronRunner" - defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") - @Suppress("UNCHECKED_CAST") - localEnvVars.putAll(env.toMap() as Map) - localEnvVars.putAll(mapOf( - "AIRBYTE_ROLE" to (System.getenv("AIRBYTE_ROLE") ?: "undefined"), - "AIRBYTE_VERSION" to env["VERSION"].toString(), - )) - } + application { + mainClass = "io.airbyte.cron.MicronautCronRunner" + defaultJvmArgs = listOf("-XX:+ExitOnOutOfMemoryError", "-XX:MaxRAMPercentage=75.0") + @Suppress("UNCHECKED_CAST") + localEnvVars.putAll(env.toMap() as Map) + localEnvVars.putAll( + mapOf( + "AIRBYTE_ROLE" to (System.getenv("AIRBYTE_ROLE") ?: "undefined"), + "AIRBYTE_VERSION" to env["VERSION"].toString(), + ), + ) + } - docker { - imageName = "cron" - } + docker { + imageName = "cron" + } } // The DuplicatesStrategy will be required while this module is mixture of kotlin and java _with_ lombok dependencies.) @@ -80,5 +83,5 @@ airbyte { // keepJavacAnnotationProcessors enabled, which causes duplicate META-INF files to be generated.) // Once lombok has been removed, this can also be removed.) tasks.withType().configureEach { - duplicatesStrategy = DuplicatesStrategy.EXCLUDE + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/ActorDefinitionService.java b/airbyte-data/src/main/java/io/airbyte/data/services/ActorDefinitionService.java index 9d4000ba62a..141cf923873 100644 --- a/airbyte-data/src/main/java/io/airbyte/data/services/ActorDefinitionService.java +++ b/airbyte-data/src/main/java/io/airbyte/data/services/ActorDefinitionService.java @@ -8,6 +8,7 @@ import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.data.exceptions.ConfigNotFoundException; +import io.airbyte.data.services.shared.ActorWorkspaceOrganizationIds; import java.io.IOException; import java.util.List; import java.util.Map; @@ -54,6 +55,8 @@ public interface ActorDefinitionService { Set getActorsWithDefaultVersionId(UUID defaultVersionId) throws IOException; + List getActorIdsForDefinition(UUID actorDefinitionId) throws IOException; + List listBreakingChangesForActorDefinition(UUID actorDefinitionId) throws IOException; void setActorDefinitionVersionSupportStates(List actorDefinitionVersionIds, ActorDefinitionVersion.SupportState supportState) diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImpl.java b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImpl.java index 6f97c2a87a1..08f6562233b 100644 --- a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImpl.java +++ b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImpl.java @@ -23,6 +23,7 @@ import io.airbyte.config.ScopeType; import io.airbyte.data.exceptions.ConfigNotFoundException; import io.airbyte.data.services.ActorDefinitionService; +import io.airbyte.data.services.shared.ActorWorkspaceOrganizationIds; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; import io.airbyte.db.instance.configs.jooq.generated.Tables; @@ -310,6 +311,18 @@ public Set getActorsWithDefaultVersionId(final UUID defaultVersionId) thro .collect(Collectors.toSet())); } + @Override + public List getActorIdsForDefinition(final UUID actorDefinitionId) throws IOException { + return database.query(ctx -> ctx.select(ACTOR.ID, ACTOR.WORKSPACE_ID, WORKSPACE.ORGANIZATION_ID) + .from(ACTOR) + .join(WORKSPACE).on(ACTOR.WORKSPACE_ID.eq(WORKSPACE.ID)) + .where(ACTOR.ACTOR_DEFINITION_ID.eq(actorDefinitionId)) + .fetch() + .stream() + .map(record -> new ActorWorkspaceOrganizationIds(record.get(ACTOR.ID), record.get(ACTOR.WORKSPACE_ID), record.get(WORKSPACE.ORGANIZATION_ID))) + .toList()); + } + @Override public void updateActorDefinitionDefaultVersionId(final UUID actorDefinitionId, final UUID versionId) throws IOException { database.query(ctx -> ctx.update(ACTOR_DEFINITION) diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ConnectionServiceJooqImpl.java b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ConnectionServiceJooqImpl.java index 466a0f6e01e..32d4a32b8a8 100644 --- a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ConnectionServiceJooqImpl.java +++ b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/ConnectionServiceJooqImpl.java @@ -480,9 +480,10 @@ private Set getEarlySyncJobsFromResult(final Result result) { } /** - * This query retrieves billable sync jobs (job status: INCOMPLETE, SUCCEEDED and CANCELLED) for - * connections that have been created in the past 7 days OR finds the first successful sync jobs for - * their corresponding connections. These results are used to mark these early syncs as free. + * This query retrieves billable sync jobs (jobs in a terminal status - succeeded, cancelled, + * failed) for connections that have been created in the past 7 days OR finds the first successful + * sync jobs for their corresponding connections. These results are used to mark these early syncs + * as free. */ private static final String EARLY_SYNC_JOB_QUERY = // Find the first successful sync job ID for every connection. @@ -500,7 +501,9 @@ private Set getEarlySyncJobsFromResult(final Result result) { + " FROM jobs j" + " LEFT JOIN connection c ON c.id = UUID(j.scope)" + " LEFT JOIN FirstSuccessfulJobIdByConnection min_j_ids ON j.id = min_j_ids.min_job_id" - + " WHERE j.status IN ('succeeded', 'incomplete', 'cancelled')" + // Consider only jobs that are in a generally accepted terminal status + // io/airbyte/persistence/job/models/JobStatus.java:23 + + " WHERE j.status IN ('succeeded', 'cancelled', 'failed')" + " AND j.config_type = 'sync'" + " AND c.id IS NOT NULL" // Keep a job if it was created within 7 days of its connection's creation, diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/shared/ActorWorkspaceOrganizationIds.java b/airbyte-data/src/main/java/io/airbyte/data/services/shared/ActorWorkspaceOrganizationIds.java new file mode 100644 index 00000000000..5cd9f2f6b0f --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/shared/ActorWorkspaceOrganizationIds.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services.shared; + +import java.util.UUID; +import javax.annotation.Nullable; + +/** + * A record that represents IDs for an actor and its associated workspace and organization. + * + * @param actorId - actor ID + * @param workspaceId - workspace ID + * @param organizationId - organization ID + */ +public record ActorWorkspaceOrganizationIds(UUID actorId, UUID workspaceId, @Nullable UUID organizationId) { + +} diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/helpers/ActorDefinitionVersionUpdater.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/helpers/ActorDefinitionVersionUpdater.kt index 13b407e4b8c..c7ddf3e4f6a 100644 --- a/airbyte-data/src/main/kotlin/io/airbyte/data/helpers/ActorDefinitionVersionUpdater.kt +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/helpers/ActorDefinitionVersionUpdater.kt @@ -10,6 +10,7 @@ import io.airbyte.config.ConfigOriginType import io.airbyte.config.ConfigResourceType import io.airbyte.config.ConfigScopeType import io.airbyte.config.DestinationConnection +import io.airbyte.config.ScopedConfiguration import io.airbyte.config.SourceConnection import io.airbyte.config.StandardDestinationDefinition import io.airbyte.config.StandardSourceDefinition @@ -18,6 +19,7 @@ import io.airbyte.config.helpers.StreamBreakingChangeScope import io.airbyte.data.services.ActorDefinitionService import io.airbyte.data.services.ConnectionService import io.airbyte.data.services.ScopedConfigurationService +import io.airbyte.data.services.shared.ConfigScopeMapWithId import io.airbyte.data.services.shared.ConnectorVersionKey import io.airbyte.featureflag.ANONYMOUS import io.airbyte.featureflag.FeatureFlagClient @@ -90,7 +92,7 @@ class ActorDefinitionVersionUpdater( } @VisibleForTesting - fun upgradeActorVersion( + internal fun upgradeActorVersion( actorId: UUID, actorDefinitionId: UUID, newVersionId: UUID, @@ -117,7 +119,7 @@ class ActorDefinitionVersionUpdater( } @VisibleForTesting - fun updateDefaultVersion( + internal fun updateDefaultVersion( actorDefinitionId: UUID, newDefaultVersion: ActorDefinitionVersion, breakingChangesForDefinition: List, @@ -128,26 +130,152 @@ class ActorDefinitionVersionUpdater( val currentDefaultVersionOpt = actorDefinitionService.getDefaultVersionForActorDefinitionIdOptional(actorDefinitionId) currentDefaultVersionOpt.ifPresent { currentDefaultVersion -> - val actorsToUpgrade = getActorsToUpgrade(currentDefaultVersion, newDefaultVersion, breakingChangesForDefinition) + val breakingChangesForUpgrade = + getBreakingChangesForUpgrade( + currentDefaultVersion.dockerImageTag, + newDefaultVersion.dockerImageTag, + breakingChangesForDefinition, + ) + + // Old: update actor.default_version_id for unaffected actors + val actorsToUpgrade = getActorsToUpgrade(currentDefaultVersion, breakingChangesForUpgrade) actorDefinitionService.setActorDefaultVersions(actorsToUpgrade.stream().toList(), newDefaultVersion.versionId) + + // New: determine which actors should NOT be upgraded, and pin those back + processBreakingChangesForUpgrade(currentDefaultVersion, breakingChangesForUpgrade) } actorDefinitionService.updateActorDefinitionDefaultVersionId(actorDefinitionId, newDefaultVersion.versionId) + + // New: for breaking changes that have been rolled back, clear old pins that may have been created + processBreakingChangePinRollbacks(actorDefinitionId, newDefaultVersion, breakingChangesForDefinition) + } + + private fun getConfigScopeMaps(actorDefinitionId: UUID): Collection { + val actorScopes = actorDefinitionService.getActorIdsForDefinition(actorDefinitionId) + return actorScopes.map { + ConfigScopeMapWithId( + it.actorId, + mapOf( + ConfigScopeType.ACTOR to it.actorId, + ConfigScopeType.WORKSPACE to it.workspaceId, + ConfigScopeType.ORGANIZATION to it.organizationId, + ), + ) + } + } + + private fun getActorIdsToPinForBreakingChange( + actorDefinitionId: UUID, + breakingChange: ActorDefinitionBreakingChange, + configScopeMaps: Collection, + ): Set { + // upgrade candidates: any actor that doesn't have a pin on it + // this must happen in order, for each BC, so when processing multiple breaking changes at once we + // determine affected actors correctly + val upgradeCandidates = getUpgradeCandidates(actorDefinitionId, configScopeMaps) + + // actors to pin: any actor from candidates (no pins) that is impacted by a breaking change + return getActorsAffectedByBreakingChange(upgradeCandidates, breakingChange) } @VisibleForTesting - fun getActorsToUpgrade( + internal fun processBreakingChangesForUpgrade( currentDefaultVersion: ActorDefinitionVersion, - newVersion: ActorDefinitionVersion, - breakingChangesForDefinition: List, + breakingChangesForUpgrade: List, + ) { + if (breakingChangesForUpgrade.isEmpty()) return + + val actorDefinitionId = currentDefaultVersion.actorDefinitionId + val configScopeMaps = getConfigScopeMaps(actorDefinitionId) + for (breakingChange in breakingChangesForUpgrade) { + val actorIdsToPin = getActorIdsToPinForBreakingChange(actorDefinitionId, breakingChange, configScopeMaps) + if (actorIdsToPin.isNotEmpty()) { + // create the pins + createBreakingChangePinsForActors(actorIdsToPin, currentDefaultVersion, breakingChange) + } + } + } + + /** + * For breaking changes that have been rolled back, clear old pins that may have been created. + * Removing the pins will cause the actors to use the new default version. + */ + @VisibleForTesting + internal fun processBreakingChangePinRollbacks( + actorDefinitionId: UUID, + newDefaultVersion: ActorDefinitionVersion, + breakingChangesForDef: List, + ) { + val rolledBackBreakingChanges = + getBreakingChangesAfterVersion( + newDefaultVersion.dockerImageTag, + breakingChangesForDef, + ) + + if (rolledBackBreakingChanges.isEmpty()) return + + val scopedConfigsToRemove = + scopedConfigurationService.listScopedConfigurationsWithOrigins( + ConnectorVersionKey.key, + ConfigResourceType.ACTOR_DEFINITION, + actorDefinitionId, + ConfigOriginType.BREAKING_CHANGE, + rolledBackBreakingChanges.map { it.version.serialize() }, + ) + + if (scopedConfigsToRemove.isNotEmpty()) { + scopedConfigurationService.deleteScopedConfigurations(scopedConfigsToRemove.map { it.id }) + } + } + + @VisibleForTesting + internal fun getUpgradeCandidates( + actorDefinitionId: UUID, + configScopeMaps: Collection, ): Set { - val breakingChangesForUpgrade: List = - getBreakingChangesForUpgrade( - currentDefaultVersion.dockerImageTag, - newVersion.dockerImageTag, - breakingChangesForDefinition, + val scopedConfigs = + scopedConfigurationService.getScopedConfigurations( + ConnectorVersionKey, + ConfigResourceType.ACTOR_DEFINITION, + actorDefinitionId, + configScopeMaps.toList(), ) + // upgrade candidates are all those actorIds that don't have a version config + return configScopeMaps.stream() + .filter { !scopedConfigs.containsKey(it.id) } + .map { it.id } + .collect(Collectors.toSet()) + } + + @VisibleForTesting + internal fun createBreakingChangePinsForActors( + actorIds: Set, + currentVersion: ActorDefinitionVersion, + breakingChange: ActorDefinitionBreakingChange, + ) { + val scopedConfigurationsToCreate = + actorIds.map { actorId -> + ScopedConfiguration() + .withId(UUID.randomUUID()) + .withKey(ConnectorVersionKey.key) + .withValue(currentVersion.versionId.toString()) + .withResourceType(ConfigResourceType.ACTOR_DEFINITION) + .withResourceId(currentVersion.actorDefinitionId) + .withScopeType(ConfigScopeType.ACTOR) + .withScopeId(actorId) + .withOriginType(ConfigOriginType.BREAKING_CHANGE) + .withOrigin(breakingChange.version.serialize()) + }.toList() + scopedConfigurationService.insertScopedConfigurations(scopedConfigurationsToCreate) + } + + @VisibleForTesting + fun getActorsToUpgrade( + currentDefaultVersion: ActorDefinitionVersion, + breakingChangesForUpgrade: List, + ): Set { val upgradeCandidates = actorDefinitionService.getActorsWithDefaultVersionId(currentDefaultVersion.versionId).toMutableSet() for (breakingChange in breakingChangesForUpgrade) { @@ -222,7 +350,7 @@ class ActorDefinitionVersionUpdater( * @return list of applicable breaking changes */ @VisibleForTesting - fun getBreakingChangesForUpgrade( + internal fun getBreakingChangesForUpgrade( currentDockerImageTag: String, dockerImageTagForUpgrade: String, breakingChangesForDef: List, @@ -242,11 +370,28 @@ class ActorDefinitionVersionUpdater( return listOf() } - return breakingChangesForDef.stream().filter { breakingChange: ActorDefinitionBreakingChange -> + return breakingChangesForDef.stream().filter { breakingChange -> ( currentVersion.lessThan(breakingChange.version) && versionToUpgradeTo.greaterThanOrEqualTo(breakingChange.version) ) - }.collect(Collectors.toList()) + }.sorted { bc1, bc2 -> bc1.version.versionCompareTo(bc2.version) }.toList() + } + + /** + * Given a new image tag, and a list of breaking changes, determine which breaking changes, if any, + * are after the new version (i.e. are not applicable to the new version). + */ + @VisibleForTesting + internal fun getBreakingChangesAfterVersion( + newImageTag: String, + breakingChangesForDef: List, + ): List { + if (breakingChangesForDef.isEmpty()) { + return listOf() + } + + val newVersion = Version(newImageTag) + return breakingChangesForDef.filter { it.version.greaterThan(newVersion) }.toList() } } diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/ConnectionTimelineEventRepository.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/ConnectionTimelineEventRepository.kt new file mode 100644 index 00000000000..26c20f01a09 --- /dev/null +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/ConnectionTimelineEventRepository.kt @@ -0,0 +1,12 @@ +package io.airbyte.data.repositories + +import io.airbyte.data.repositories.entities.ConnectionTimelineEvent +import io.micronaut.data.jdbc.annotation.JdbcRepository +import io.micronaut.data.model.query.builder.sql.Dialect +import io.micronaut.data.repository.PageableRepository +import java.util.UUID + +@JdbcRepository(dialect = Dialect.POSTGRES, dataSource = "config") +interface ConnectionTimelineEventRepository : PageableRepository { + fun findByConnectionId(connectionId: UUID): List +} diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/UserInvitationRepository.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/UserInvitationRepository.kt index f033baad474..ce3a19632f9 100644 --- a/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/UserInvitationRepository.kt +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/UserInvitationRepository.kt @@ -18,4 +18,11 @@ interface UserInvitationRepository : PageableRepository { scopeType: EntityScopeType, scopeId: UUID, ): List + + fun findByStatusAndScopeTypeAndScopeIdAndInvitedEmail( + status: EntityInvitationStatus, + scopeType: EntityScopeType, + scopeId: UUID, + invitedEmail: String, + ): List } diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/entities/ConnectionTimelineEvent.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/entities/ConnectionTimelineEvent.kt new file mode 100644 index 00000000000..ea80b04629a --- /dev/null +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/entities/ConnectionTimelineEvent.kt @@ -0,0 +1,24 @@ +package io.airbyte.data.repositories.entities + +import io.micronaut.data.annotation.AutoPopulated +import io.micronaut.data.annotation.DateCreated +import io.micronaut.data.annotation.Id +import io.micronaut.data.annotation.MappedEntity +import io.micronaut.data.annotation.TypeDef +import io.micronaut.data.model.DataType +import java.time.OffsetDateTime +import java.util.UUID + +@MappedEntity("connection_timeline_event") +data class ConnectionTimelineEvent( + @field:Id + @AutoPopulated + var id: UUID? = null, + var connectionId: UUID, + var userId: UUID? = null, + var eventType: String, + @field:TypeDef(type = DataType.JSON) + var summary: String? = null, + @DateCreated + var createdAt: java.time.OffsetDateTime? = null, +) diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/entities/UserInvitation.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/entities/UserInvitation.kt index 521dab53d01..3bc0dacd5c8 100644 --- a/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/entities/UserInvitation.kt +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/repositories/entities/UserInvitation.kt @@ -3,8 +3,10 @@ package io.airbyte.data.repositories.entities import io.airbyte.db.instance.configs.jooq.generated.enums.InvitationStatus import io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType import io.airbyte.db.instance.configs.jooq.generated.enums.ScopeType +import io.micronaut.core.annotation.Nullable import io.micronaut.data.annotation.AutoPopulated import io.micronaut.data.annotation.DateCreated +import io.micronaut.data.annotation.DateUpdated import io.micronaut.data.annotation.Id import io.micronaut.data.annotation.MappedEntity import io.micronaut.data.annotation.TypeDef @@ -19,6 +21,8 @@ data class UserInvitation( var inviteCode: String, var inviterUserId: UUID, var invitedEmail: String, + @Nullable + var acceptedByUserId: UUID? = null, var scopeId: UUID, @field:TypeDef(type = DataType.OBJECT) var scopeType: ScopeType, @@ -28,6 +32,7 @@ data class UserInvitation( var status: InvitationStatus, @DateCreated var createdAt: java.time.OffsetDateTime? = null, - @DateCreated + @DateUpdated var updatedAt: java.time.OffsetDateTime? = null, + var expiresAt: java.time.OffsetDateTime, ) diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/services/ConnectionTimelineEventService.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/services/ConnectionTimelineEventService.kt new file mode 100644 index 00000000000..43e35a4c7c4 --- /dev/null +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/services/ConnectionTimelineEventService.kt @@ -0,0 +1,7 @@ +package io.airbyte.data.services + +import io.airbyte.data.repositories.entities.ConnectionTimelineEvent + +interface ConnectionTimelineEventService { + fun writeEvent(event: ConnectionTimelineEvent): ConnectionTimelineEvent +} diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/services/UserInvitationService.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/services/UserInvitationService.kt index 4938c830576..fddff4d05e4 100644 --- a/airbyte-data/src/main/kotlin/io/airbyte/data/services/UserInvitationService.kt +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/services/UserInvitationService.kt @@ -24,14 +24,16 @@ interface UserInvitationService { /** * Create a new user invitation. */ + @Throws(InvitationDuplicateException::class) fun createUserInvitation(invitation: UserInvitation): UserInvitation /** * Accept a user invitation and create resulting permission record. */ + @Throws(InvitationStatusUnexpectedException::class) fun acceptUserInvitation( inviteCode: String, - invitedUserId: UUID, + acceptingUserId: UUID, ): UserInvitation /** @@ -45,5 +47,18 @@ interface UserInvitationService { /** * Cancel a user invitation. */ + @Throws(InvitationStatusUnexpectedException::class) fun cancelUserInvitation(inviteCode: String): UserInvitation } + +/** + * Exception thrown when an operation on an invitation cannot be performed because it has an + * unexpected status. For instance, trying to accept an invitation that is not pending. + */ +class InvitationStatusUnexpectedException(message: String) : Exception(message) + +/** + * Exception thrown when trying to create a duplicate invitation, ie creating new invitation with + * the same email and scope as an existing pending invitation. + */ +class InvitationDuplicateException(message: String) : Exception(message) diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/ConnectionTimelineEventServiceImpl.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/ConnectionTimelineEventServiceImpl.kt new file mode 100644 index 00000000000..9c73e088b38 --- /dev/null +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/ConnectionTimelineEventServiceImpl.kt @@ -0,0 +1,13 @@ +package io.airbyte.data.services.impls.data + +import io.airbyte.data.repositories.ConnectionTimelineEventRepository +import io.airbyte.data.repositories.entities.ConnectionTimelineEvent +import io.airbyte.data.services.ConnectionTimelineEventService +import jakarta.inject.Singleton + +@Singleton +class ConnectionTimelineEventServiceImpl(private val repository: ConnectionTimelineEventRepository) : ConnectionTimelineEventService { + override fun writeEvent(event: ConnectionTimelineEvent): ConnectionTimelineEvent { + return repository.save(event) + } +} diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/UserInvitationServiceDataImpl.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/UserInvitationServiceDataImpl.kt index 6910ecc97b7..dfcb6def191 100644 --- a/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/UserInvitationServiceDataImpl.kt +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/UserInvitationServiceDataImpl.kt @@ -1,19 +1,23 @@ package io.airbyte.data.services.impls.data import io.airbyte.config.ConfigSchema -import io.airbyte.config.InvitationStatus -import io.airbyte.config.Permission import io.airbyte.config.ScopeType import io.airbyte.config.UserInvitation import io.airbyte.data.exceptions.ConfigNotFoundException import io.airbyte.data.repositories.PermissionRepository import io.airbyte.data.repositories.UserInvitationRepository +import io.airbyte.data.repositories.entities.Permission +import io.airbyte.data.services.InvitationDuplicateException +import io.airbyte.data.services.InvitationStatusUnexpectedException import io.airbyte.data.services.UserInvitationService import io.airbyte.data.services.impls.data.mappers.EntityInvitationStatus +import io.airbyte.data.services.impls.data.mappers.EntityScopeType +import io.airbyte.data.services.impls.data.mappers.EntityUserInvitation import io.airbyte.data.services.impls.data.mappers.toConfigModel import io.airbyte.data.services.impls.data.mappers.toEntity import io.micronaut.transaction.annotation.Transactional import jakarta.inject.Singleton +import java.time.OffsetDateTime import java.util.UUID @Singleton @@ -39,40 +43,61 @@ open class UserInvitationServiceDataImpl( } override fun createUserInvitation(invitation: UserInvitation): UserInvitation { + // throw an exception if a pending invitation already exists for the same email and scope + val existingInvitations = + userInvitationRepository.findByStatusAndScopeTypeAndScopeIdAndInvitedEmail( + EntityInvitationStatus.pending, + invitation.scopeType.toEntity(), + invitation.scopeId, + invitation.invitedEmail, + ) + if (existingInvitations.isNotEmpty()) { + throw InvitationDuplicateException( + "A pending invitation already exists for InvitedEmail: ${invitation.invitedEmail}, ScopeType: ${invitation.scopeType} " + + "and ScopeId: ${invitation.scopeId}", + ) + } + return userInvitationRepository.save(invitation.toEntity()).toConfigModel() } @Transactional("config") override fun acceptUserInvitation( inviteCode: String, - invitedUserId: UUID, + acceptingUserId: UUID, ): UserInvitation { // fetch the invitation by code val invitation = userInvitationRepository.findByInviteCode(inviteCode).orElseThrow { ConfigNotFoundException(ConfigSchema.USER_INVITATION, inviteCode) - }.toConfigModel() + } - if (invitation.status != InvitationStatus.PENDING) { - throw IllegalStateException("Invitation status is not pending: ${invitation.status}") + // mark the invitation status as expired if expiresAt is in the past + if (invitation.expiresAt.isBefore(OffsetDateTime.now())) { + invitation.status = EntityInvitationStatus.expired + userInvitationRepository.update(invitation) } + // throw an exception if the invitation is not pending. Note that this will also + // catch the case where the invitation is expired. + throwIfNotPending(invitation) + // create a new permission record according to the invitation - val permission = - Permission().apply { - userId = invitedUserId - permissionType = invitation.permissionType - when (invitation.scopeType) { - ScopeType.ORGANIZATION -> organizationId = invitation.scopeId - ScopeType.WORKSPACE -> workspaceId = invitation.scopeId - else -> throw IllegalStateException("Unknown scope type: ${invitation.scopeType}") - } + Permission( + id = UUID.randomUUID(), + userId = acceptingUserId, + permissionType = invitation.permissionType, + ).apply { + when (invitation.scopeType) { + EntityScopeType.organization -> organizationId = invitation.scopeId + EntityScopeType.workspace -> workspaceId = invitation.scopeId } - permissionRepository.save(permission.toEntity()) + }.let { permissionRepository.save(it) } - // update the invitation status to accepted - invitation.status = InvitationStatus.ACCEPTED - val updatedInvitation = userInvitationRepository.update(invitation.toEntity()) + // mark the invitation as accepted + invitation.status = EntityInvitationStatus.accepted + invitation.acceptedByUserId = acceptingUserId + val updatedInvitation = userInvitationRepository.update(invitation) return updatedInvitation.toConfigModel() } @@ -85,6 +110,25 @@ open class UserInvitationServiceDataImpl( } override fun cancelUserInvitation(inviteCode: String): UserInvitation { - TODO("Not yet implemented") + val invitation = + userInvitationRepository.findByInviteCode(inviteCode).orElseThrow { + ConfigNotFoundException(ConfigSchema.USER_INVITATION, inviteCode) + } + + throwIfNotPending(invitation) + + invitation.status = EntityInvitationStatus.cancelled + val updatedInvitation = userInvitationRepository.update(invitation) + + return updatedInvitation.toConfigModel() + } + + private fun throwIfNotPending(invitation: EntityUserInvitation) { + if (invitation.status != EntityInvitationStatus.pending) { + throw InvitationStatusUnexpectedException( + "Expected invitation for ScopeType: ${invitation.scopeType} and ScopeId: ${invitation.scopeId} to " + + "be PENDING, but instead it had Status: ${invitation.status}", + ) + } } } diff --git a/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/mappers/UserInvitationMapper.kt b/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/mappers/UserInvitationMapper.kt index 1f1f09c7781..62ed56f01da 100644 --- a/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/mappers/UserInvitationMapper.kt +++ b/airbyte-data/src/main/kotlin/io/airbyte/data/services/impls/data/mappers/UserInvitationMapper.kt @@ -32,6 +32,7 @@ fun EntityInvitationStatus.toConfigModel(): ModelInvitationStatus { EntityInvitationStatus.accepted -> ModelInvitationStatus.ACCEPTED EntityInvitationStatus.cancelled -> ModelInvitationStatus.CANCELLED EntityInvitationStatus.declined -> ModelInvitationStatus.DECLINED + EntityInvitationStatus.expired -> ModelInvitationStatus.EXPIRED } } @@ -41,6 +42,7 @@ fun ModelInvitationStatus.toEntity(): EntityInvitationStatus { ModelInvitationStatus.ACCEPTED -> EntityInvitationStatus.accepted ModelInvitationStatus.CANCELLED -> EntityInvitationStatus.cancelled ModelInvitationStatus.DECLINED -> EntityInvitationStatus.declined + ModelInvitationStatus.EXPIRED -> EntityInvitationStatus.expired } } @@ -50,12 +52,14 @@ fun EntityUserInvitation.toConfigModel(): ModelUserInvitation { .withInviteCode(this.inviteCode) .withInviterUserId(this.inviterUserId) .withInvitedEmail(this.invitedEmail) + .withAcceptedByUserId(this.acceptedByUserId) .withScopeId(this.scopeId) .withScopeType(this.scopeType.toConfigModel()) .withPermissionType(this.permissionType.toConfigModel()) .withStatus(this.status.toConfigModel()) .withCreatedAt(this.createdAt?.toEpochSecond()) .withUpdatedAt(this.updatedAt?.toEpochSecond()) + .withExpiresAt(this.expiresAt.toEpochSecond()) } fun ModelUserInvitation.toEntity(): EntityUserInvitation { @@ -64,11 +68,13 @@ fun ModelUserInvitation.toEntity(): EntityUserInvitation { inviteCode = this.inviteCode, inviterUserId = this.inviterUserId, invitedEmail = this.invitedEmail, + acceptedByUserId = this.acceptedByUserId, scopeId = this.scopeId, scopeType = this.scopeType.toEntity(), permissionType = this.permissionType.toEntity(), status = this.status.toEntity(), createdAt = this.createdAt?.let { OffsetDateTime.ofInstant(Instant.ofEpochSecond(it), ZoneOffset.UTC) }, updatedAt = this.updatedAt?.let { OffsetDateTime.ofInstant(Instant.ofEpochSecond(it), ZoneOffset.UTC) }, + expiresAt = OffsetDateTime.ofInstant(Instant.ofEpochSecond(this.expiresAt), ZoneOffset.UTC), ) } diff --git a/airbyte-data/src/test/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImplTest.java b/airbyte-data/src/test/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImplTest.java index 51607f6be46..b89ef7bbac4 100644 --- a/airbyte-data/src/test/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImplTest.java +++ b/airbyte-data/src/test/java/io/airbyte/data/services/impls/jooq/ActorDefinitionServiceJooqImplTest.java @@ -22,6 +22,7 @@ import io.airbyte.data.services.ScopedConfigurationService; import io.airbyte.data.services.SecretPersistenceConfigService; import io.airbyte.data.services.SourceService; +import io.airbyte.data.services.shared.ActorWorkspaceOrganizationIds; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HeartbeatMaxSecondsBetweenMessages; import io.airbyte.featureflag.TestClient; @@ -36,11 +37,12 @@ class ActorDefinitionServiceJooqImplTest extends BaseConfigDatabaseTest { + private JooqTestDbSetupHelper jooqTestDbSetupHelper; private SourceService sourceService; private ActorDefinitionServiceJooqImpl actorDefinitionService; @BeforeEach - void setUp() { + void setUp() throws JsonValidationException, ConfigNotFoundException, IOException { this.actorDefinitionService = new ActorDefinitionServiceJooqImpl(database); final FeatureFlagClient featureFlagClient = mock(TestClient.class); @@ -55,13 +57,13 @@ void setUp() { new ActorDefinitionVersionUpdater(featureFlagClient, connectionService, actorDefinitionService, scopedConfigurationService); this.sourceService = new SourceServiceJooqImpl(database, featureFlagClient, secretsRepositoryReader, secretsRepositoryWriter, secretPersistenceConfigService, connectionService, actorDefinitionVersionUpdater); - } - @Test - void testSetActorDefaultVersions() throws JsonValidationException, ConfigNotFoundException, IOException { - final JooqTestDbSetupHelper jooqTestDbSetupHelper = new JooqTestDbSetupHelper(); + jooqTestDbSetupHelper = new JooqTestDbSetupHelper(); jooqTestDbSetupHelper.setupForVersionUpgradeTest(); + } + @Test + void testSetActorDefaultVersions() throws IOException { final UUID actorId = jooqTestDbSetupHelper.getSource().getSourceId(); final UUID otherActorId = UUID.randomUUID(); final SourceConnection otherSource = Jsons.clone(jooqTestDbSetupHelper.getSource()).withSourceId(otherActorId); @@ -82,10 +84,7 @@ void testSetActorDefaultVersions() throws JsonValidationException, ConfigNotFoun } @Test - void testGetActorsWithDefaultVersionId() throws JsonValidationException, ConfigNotFoundException, IOException { - final JooqTestDbSetupHelper jooqTestDbSetupHelper = new JooqTestDbSetupHelper(); - jooqTestDbSetupHelper.setupForVersionUpgradeTest(); - + void testGetActorsWithDefaultVersionId() throws IOException { final UUID actorId = jooqTestDbSetupHelper.getSource().getSourceId(); final Set actorIds = actorDefinitionService.getActorsWithDefaultVersionId(jooqTestDbSetupHelper.getInitialSourceDefaultVersionId()); assertEquals(Set.of(actorId), actorIds); @@ -93,9 +92,6 @@ void testGetActorsWithDefaultVersionId() throws JsonValidationException, ConfigN @Test void updateActorDefinitionDefaultVersionId() throws JsonValidationException, ConfigNotFoundException, IOException { - final JooqTestDbSetupHelper jooqTestDbSetupHelper = new JooqTestDbSetupHelper(); - jooqTestDbSetupHelper.setupForVersionUpgradeTest(); - final UUID actorDefinitionId = jooqTestDbSetupHelper.getSourceDefinition().getSourceDefinitionId(); final StandardSourceDefinition sourceDefinition = sourceService.getStandardSourceDefinition(actorDefinitionId); assertEquals(sourceDefinition.getDefaultVersionId(), jooqTestDbSetupHelper.getInitialSourceDefaultVersionId()); @@ -110,4 +106,16 @@ void updateActorDefinitionDefaultVersionId() throws JsonValidationException, Con assertEquals(updatedSourceDefinition.getDefaultVersionId(), newVersion.getVersionId()); } + @Test + void testGetActorIdsForDefinition() throws IOException { + final UUID actorDefinitionId = jooqTestDbSetupHelper.getSourceDefinition().getSourceDefinitionId(); + + final UUID sourceActorId = jooqTestDbSetupHelper.getSource().getSourceId(); + final UUID workspaceId = jooqTestDbSetupHelper.getWorkspace().getWorkspaceId(); + final UUID organizationId = jooqTestDbSetupHelper.getOrganization().getOrganizationId(); + + final List actorIds = actorDefinitionService.getActorIdsForDefinition(actorDefinitionId); + assertEquals(List.of(new ActorWorkspaceOrganizationIds(sourceActorId, workspaceId, organizationId)), actorIds); + } + } diff --git a/airbyte-data/src/test/java/io/airbyte/data/services/impls/jooq/JooqTestDbSetupHelper.java b/airbyte-data/src/test/java/io/airbyte/data/services/impls/jooq/JooqTestDbSetupHelper.java index 2b212d8af09..1eadc9bbc63 100644 --- a/airbyte-data/src/test/java/io/airbyte/data/services/impls/jooq/JooqTestDbSetupHelper.java +++ b/airbyte-data/src/test/java/io/airbyte/data/services/impls/jooq/JooqTestDbSetupHelper.java @@ -15,6 +15,7 @@ import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.DestinationConnection; import io.airbyte.config.Geography; +import io.airbyte.config.Organization; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; @@ -45,11 +46,15 @@ public class JooqTestDbSetupHelper extends BaseConfigDatabaseTest { private final SourceServiceJooqImpl sourceServiceJooqImpl; private final DestinationServiceJooqImpl destinationServiceJooqImpl; private final WorkspaceServiceJooqImpl workspaceServiceJooqImpl; + private final OrganizationServiceJooqImpl organizationServiceJooqImpl; private final TestClient featureFlagClient; + private final UUID ORGANIZATION_ID = UUID.randomUUID(); private final UUID WORKSPACE_ID = UUID.randomUUID(); private final UUID SOURCE_DEFINITION_ID = UUID.randomUUID(); private final UUID DESTINATION_DEFINITION_ID = UUID.randomUUID(); @Getter + private Organization organization; + @Getter private StandardWorkspace workspace; @Getter private StandardSourceDefinition sourceDefinition; @@ -100,9 +105,14 @@ public JooqTestDbSetupHelper() { secretsRepositoryReader, secretsRepositoryWriter, secretPersistenceConfigService); + this.organizationServiceJooqImpl = new OrganizationServiceJooqImpl(database); } public void setupForVersionUpgradeTest() throws IOException, JsonValidationException, ConfigNotFoundException { + // Create org + organization = createBaseOrganization(); + organizationServiceJooqImpl.writeOrganization(organization); + // Create workspace workspace = createBaseWorkspace(); workspaceServiceJooqImpl.writeStandardWorkspaceNoSecrets(createBaseWorkspace()); @@ -181,9 +191,19 @@ private SourceConnection createBaseSourceActor() { .withName("source"); } + private Organization createBaseOrganization() { + return new Organization() + .withOrganizationId(ORGANIZATION_ID) + .withName("organization") + .withEmail("org@airbyte.io") + .withPba(false) + .withOrgLevelBilling(false); + } + private StandardWorkspace createBaseWorkspace() { return new StandardWorkspace() .withWorkspaceId(WORKSPACE_ID) + .withOrganizationId(ORGANIZATION_ID) .withName("default") .withSlug("workspace-slug") .withInitialSetupComplete(false) diff --git a/airbyte-data/src/test/kotlin/io/airbyte/data/helpers/ActorDefinitionVersionUpdaterTest.kt b/airbyte-data/src/test/kotlin/io/airbyte/data/helpers/ActorDefinitionVersionUpdaterTest.kt index 72af6da07bf..3021f5c99b4 100644 --- a/airbyte-data/src/test/kotlin/io/airbyte/data/helpers/ActorDefinitionVersionUpdaterTest.kt +++ b/airbyte-data/src/test/kotlin/io/airbyte/data/helpers/ActorDefinitionVersionUpdaterTest.kt @@ -13,6 +13,8 @@ import io.airbyte.config.persistence.MockData import io.airbyte.data.services.ActorDefinitionService import io.airbyte.data.services.ConnectionService import io.airbyte.data.services.ScopedConfigurationService +import io.airbyte.data.services.shared.ActorWorkspaceOrganizationIds +import io.airbyte.data.services.shared.ConfigScopeMapWithId import io.airbyte.data.services.shared.ConnectorVersionKey import io.airbyte.featureflag.ANONYMOUS import io.airbyte.featureflag.TestClient @@ -21,8 +23,11 @@ import io.airbyte.featureflag.Workspace import io.mockk.clearAllMocks import io.mockk.every import io.mockk.mockk +import io.mockk.slot +import io.mockk.verify import io.mockk.verifyAll import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -97,6 +102,17 @@ internal class ActorDefinitionVersionUpdaterTest { Arguments.of("2.0.0", "2.0.0", listOf()), ) } + + @JvmStatic + fun getBreakingChangesAfterVersionMethodSource(): List { + return listOf( + Arguments.of("0.1.0", listOf("1.0.0", "2.0.0", "3.0.0")), + Arguments { arrayOf("1.0.0", listOf("2.0.0", "3.0.0")) }, + Arguments { arrayOf("2.0.0", listOf("3.0.0")) }, + Arguments { arrayOf("3.0.0", listOf()) }, + Arguments { arrayOf("4.0.0", listOf()) }, + ) + } } @BeforeEach @@ -116,14 +132,26 @@ internal class ActorDefinitionVersionUpdaterTest { } returns Optional.of(DEFAULT_VERSION) val actorId = UUID.randomUUID() + val workspaceId = UUID.randomUUID() + val organizationId = UUID.randomUUID() + every { actorDefinitionService.getActorsWithDefaultVersionId(DEFAULT_VERSION.versionId) } returns setOf(actorId) + every { + actorDefinitionService.getActorIdsForDefinition(ACTOR_DEFINITION_ID) + } returns listOf(ActorWorkspaceOrganizationIds(actorId, workspaceId, organizationId)) + every { connectionService.actorSyncsAnyListedStream(actorId, listOf("affected_stream")) } returns actorIsInBreakingChangeScope + val configsToWriteSlot = slot>() + every { + scopedConfigurationService.insertScopedConfigurations(capture(configsToWriteSlot)) + } returns listOf() + actorDefinitionVersionUpdater.updateDefaultVersion( ACTOR_DEFINITION_ID, NEW_VERSION, @@ -134,7 +162,23 @@ internal class ActorDefinitionVersionUpdaterTest { featureFlagClient.boolVariation(UseBreakingChangeScopes, Workspace(ANONYMOUS)) actorDefinitionService.getDefaultVersionForActorDefinitionIdOptional(ACTOR_DEFINITION_ID) actorDefinitionService.getActorsWithDefaultVersionId(DEFAULT_VERSION.versionId) + actorDefinitionService.getActorIdsForDefinition(ACTOR_DEFINITION_ID) connectionService.actorSyncsAnyListedStream(actorId, listOf("affected_stream")) + scopedConfigurationService.getScopedConfigurations( + ConnectorVersionKey, + ConfigResourceType.ACTOR_DEFINITION, + ACTOR_DEFINITION_ID, + listOf( + ConfigScopeMapWithId( + actorId, + mapOf( + ConfigScopeType.ACTOR to actorId, + ConfigScopeType.WORKSPACE to workspaceId, + ConfigScopeType.ORGANIZATION to organizationId, + ), + ), + ), + ) // Destination definition should always get the new version actorDefinitionService.updateActorDefinitionDefaultVersionId(ACTOR_DEFINITION_ID, NEW_VERSION.versionId) @@ -142,11 +186,35 @@ internal class ActorDefinitionVersionUpdaterTest { if (actorIsInBreakingChangeScope) { // Assert actor is not updated actorDefinitionService.setActorDefaultVersions(listOf(), NEW_VERSION.versionId) + + // Assert pins are created + scopedConfigurationService.insertScopedConfigurations(any()) + assertEquals(1, configsToWriteSlot.captured.size) + + val capturedConfig = configsToWriteSlot.captured[0] + assertEquals( + ScopedConfiguration() + .withKey(ConnectorVersionKey.key) + .withValue(DEFAULT_VERSION.versionId.toString()) + .withResourceType(ConfigResourceType.ACTOR_DEFINITION) + .withResourceId(ACTOR_DEFINITION_ID) + .withScopeType(ConfigScopeType.ACTOR) + .withScopeId(actorId) + .withOriginType(ConfigOriginType.BREAKING_CHANGE) + .withOrigin(STREAM_SCOPED_BREAKING_CHANGE.version.serialize()), + capturedConfig.withId(null), + ) } else { // Assert actor is upgraded to the new version actorDefinitionService.setActorDefaultVersions(listOf(actorId), NEW_VERSION.versionId) } } + + if (!actorIsInBreakingChangeScope) { + verify(exactly = 0) { + scopedConfigurationService.insertScopedConfigurations(any()) + } + } } @ParameterizedTest @@ -161,7 +229,7 @@ internal class ActorDefinitionVersionUpdaterTest { actorDefinitionService.getActorsWithDefaultVersionId(DEFAULT_VERSION.versionId) } returns setOf(actorIdOnInitialVersion) - val actorsToUpgrade = actorDefinitionVersionUpdater.getActorsToUpgrade(DEFAULT_VERSION, NEW_VERSION, listOf()) + val actorsToUpgrade = actorDefinitionVersionUpdater.getActorsToUpgrade(DEFAULT_VERSION, listOf()) // All actors should get upgraded assertEquals(setOf(actorIdOnInitialVersion), actorsToUpgrade) @@ -193,7 +261,6 @@ internal class ActorDefinitionVersionUpdaterTest { val actorsToUpgrade = actorDefinitionVersionUpdater.getActorsToUpgrade( DEFAULT_VERSION, - NEW_VERSION, listOf(STREAM_SCOPED_BREAKING_CHANGE), ) @@ -394,4 +461,293 @@ internal class ActorDefinitionVersionUpdaterTest { actorDefinitionService.setActorDefaultVersion(actorId, newVersionId) } } + + @Test + fun testProcessBreakingChangesForUpgrade() { + every { + featureFlagClient.boolVariation(UseBreakingChangeScopes, Workspace(ANONYMOUS)) + } returns true + + val pinnedActorId = UUID.randomUUID() + val withImpactedStreamActorId = UUID.randomUUID() + val noImpactedStreamActorId = UUID.randomUUID() + val noImpactedStreamActorId2 = UUID.randomUUID() + + val actors = + listOf( + ActorWorkspaceOrganizationIds(pinnedActorId, UUID.randomUUID(), UUID.randomUUID()), + ActorWorkspaceOrganizationIds(withImpactedStreamActorId, UUID.randomUUID(), UUID.randomUUID()), + ActorWorkspaceOrganizationIds(noImpactedStreamActorId, UUID.randomUUID(), UUID.randomUUID()), + ActorWorkspaceOrganizationIds(noImpactedStreamActorId2, UUID.randomUUID(), UUID.randomUUID()), + ) + + val currentVersion = DEFAULT_VERSION + val limitedScopeBreakingChange = STREAM_SCOPED_BREAKING_CHANGE + val breakingChange = MockData.actorDefinitionBreakingChange("3.0.0") + val breakingChangesForUpgrade = listOf(limitedScopeBreakingChange, breakingChange) + + every { + actorDefinitionService.getActorIdsForDefinition(ACTOR_DEFINITION_ID) + } returns actors + + val scopeMaps = actors.map { idsToConfigScopeMap(it) } + + // Setup: as we process the breaking changes, the pinned actors returned will include actors pinned due to the prior breaking change + every { + scopedConfigurationService.getScopedConfigurations( + ConnectorVersionKey, + ConfigResourceType.ACTOR_DEFINITION, + ACTOR_DEFINITION_ID, + scopeMaps, + ) + } returnsMany + listOf( + mapOf( + pinnedActorId to ScopedConfiguration(), + ), + mapOf( + pinnedActorId to ScopedConfiguration(), + withImpactedStreamActorId to ScopedConfiguration(), + ), + mapOf( + pinnedActorId to ScopedConfiguration(), + withImpactedStreamActorId to ScopedConfiguration(), + noImpactedStreamActorId to ScopedConfiguration(), + noImpactedStreamActorId2 to ScopedConfiguration(), + ), + ) + + // Setup: For the limited-impact breaking change, only mock that the targeted actor is syncing the affected stream + every { + connectionService.actorSyncsAnyListedStream(any(), any()) + } returns false + + every { + connectionService.actorSyncsAnyListedStream(withImpactedStreamActorId, listOf("affected_stream")) + } returns true + + // Collect written configs to perform assertions + val capturedConfigsToWrite = mutableListOf>() + every { + scopedConfigurationService.insertScopedConfigurations(capture(capturedConfigsToWrite)) + } returns listOf() + + // Act: call method under test + actorDefinitionVersionUpdater.processBreakingChangesForUpgrade( + currentVersion, + breakingChangesForUpgrade, + ) + + verify { + actorDefinitionService.getActorIdsForDefinition(ACTOR_DEFINITION_ID) + connectionService.actorSyncsAnyListedStream(noImpactedStreamActorId, listOf("affected_stream")) + } + + // Assert: we get pinned actors and insert new pins for each processed breaking change (2) + verify(exactly = 2) { + scopedConfigurationService.getScopedConfigurations( + ConnectorVersionKey, + ConfigResourceType.ACTOR_DEFINITION, + ACTOR_DEFINITION_ID, + scopeMaps, + ) + scopedConfigurationService.insertScopedConfigurations(any()) + } + + assertEquals(2, capturedConfigsToWrite.size) + + // Assert: limited-impact breaking change should pin the actor with the affected stream + val configsForScopedBC = capturedConfigsToWrite[0] + assertEquals(1, configsForScopedBC.size) + val expectedConfig1 = buildBreakingChangeScopedConfig(withImpactedStreamActorId, limitedScopeBreakingChange) + assertEquals(expectedConfig1, configsForScopedBC[0].withId(null)) + + // Assert: breaking change should pin all remaining unpinned actors + val configsForGlobalBC = capturedConfigsToWrite[1].sortedBy { it.scopeId } + assertEquals(2, configsForGlobalBC.size) + val sortedExpectedIds = listOf(noImpactedStreamActorId, noImpactedStreamActorId2).sorted() + val expectedConfig2 = buildBreakingChangeScopedConfig(sortedExpectedIds[0], breakingChange) + assertEquals(expectedConfig2, configsForGlobalBC[0].withId(null)) + + val expectedConfig3 = buildBreakingChangeScopedConfig(sortedExpectedIds[1], breakingChange) + assertEquals(expectedConfig3, configsForGlobalBC[1].withId(null)) + } + + @Test + fun testGetUpgradeCandidates() { + val pinnedActorId = UUID.randomUUID() + val pinnedActorId2 = UUID.randomUUID() + val unpinnedActorId = UUID.randomUUID() + val unpinnedActorId2 = UUID.randomUUID() + val configScopeMaps = + listOf( + ConfigScopeMapWithId(pinnedActorId, mapOf()), + ConfigScopeMapWithId(pinnedActorId2, mapOf()), + ConfigScopeMapWithId(unpinnedActorId, mapOf()), + ConfigScopeMapWithId(unpinnedActorId2, mapOf()), + ) + + every { + scopedConfigurationService.getScopedConfigurations( + ConnectorVersionKey, + ConfigResourceType.ACTOR_DEFINITION, + ACTOR_DEFINITION_ID, + configScopeMaps, + ) + } returns + mapOf( + pinnedActorId to ScopedConfiguration(), + pinnedActorId2 to ScopedConfiguration(), + ) + + val upgradeCandidates = actorDefinitionVersionUpdater.getUpgradeCandidates(ACTOR_DEFINITION_ID, configScopeMaps) + + assertEquals(2, upgradeCandidates.size) + assertEquals(setOf(unpinnedActorId, unpinnedActorId2), upgradeCandidates) + + verifyAll { + scopedConfigurationService.getScopedConfigurations( + ConnectorVersionKey, + ConfigResourceType.ACTOR_DEFINITION, + ACTOR_DEFINITION_ID, + configScopeMaps, + ) + } + } + + @Test + fun testCreateBreakingChangePinsForActors() { + val actorIds = setOf(UUID.randomUUID(), UUID.randomUUID()) + + val scopedConfigsCapture = slot>() + + every { + scopedConfigurationService.insertScopedConfigurations(capture(scopedConfigsCapture)) + } returns listOf() + + actorDefinitionVersionUpdater.createBreakingChangePinsForActors(actorIds, DEFAULT_VERSION, STREAM_SCOPED_BREAKING_CHANGE) + + verify(exactly = 1) { + scopedConfigurationService.insertScopedConfigurations(any()) + } + + assertEquals(2, scopedConfigsCapture.captured.size) + for (actorId in actorIds) { + val capturedConfig = scopedConfigsCapture.captured.find { it.scopeId == actorId } + assertNotNull(capturedConfig) + assertNotNull(capturedConfig!!.id) + + val expectedConfig = buildBreakingChangeScopedConfig(actorId, STREAM_SCOPED_BREAKING_CHANGE) + assertEquals(expectedConfig, capturedConfig.withId(null)) + } + } + + @ParameterizedTest + @MethodSource("getBreakingChangesAfterVersionMethodSource") + fun testGetBreakingChangesAfterVersion( + versionTag: String, + expectedBreakingChanges: List, + ) { + val breakingChanges = + listOf( + MockData.actorDefinitionBreakingChange("1.0.0"), + MockData.actorDefinitionBreakingChange("2.0.0"), + MockData.actorDefinitionBreakingChange("3.0.0"), + ) + + val actualBreakingChanges = + actorDefinitionVersionUpdater.getBreakingChangesAfterVersion( + versionTag, + breakingChanges, + ).map { it.version.serialize() }.toList() + + assertEquals(expectedBreakingChanges, actualBreakingChanges) + } + + @Test + fun testGetBreakingChangesAfterVersionWithNoBreakingChanges() { + val actualBreakingChanges = + actorDefinitionVersionUpdater.getBreakingChangesAfterVersion( + "1.0.0", + listOf(), + ) + + assertEquals(listOf(), actualBreakingChanges) + } + + @Test + fun testProcessBreakingChangePinRollbacks() { + val oldBC = MockData.actorDefinitionBreakingChange("1.0.0") + val currentVersionBC = MockData.actorDefinitionBreakingChange("2.0.0") + val rolledBackBC = MockData.actorDefinitionBreakingChange("3.0.0") + + val allBreakingChanges = listOf(oldBC, currentVersionBC, rolledBackBC) + val idsPinnedForV3 = listOf(UUID.randomUUID(), UUID.randomUUID()) + + every { + scopedConfigurationService.listScopedConfigurationsWithOrigins( + ConnectorVersionKey.key, + ConfigResourceType.ACTOR_DEFINITION, + ACTOR_DEFINITION_ID, + ConfigOriginType.BREAKING_CHANGE, + listOf(rolledBackBC.version.serialize()), + ) + } returns idsPinnedForV3.map { buildBreakingChangeScopedConfig(it, rolledBackBC).withId(it) } + + actorDefinitionVersionUpdater.processBreakingChangePinRollbacks(ACTOR_DEFINITION_ID, NEW_VERSION, allBreakingChanges) + + verifyAll { + scopedConfigurationService.listScopedConfigurationsWithOrigins( + ConnectorVersionKey.key, + ConfigResourceType.ACTOR_DEFINITION, + ACTOR_DEFINITION_ID, + ConfigOriginType.BREAKING_CHANGE, + listOf(rolledBackBC.version.serialize()), + ) + + scopedConfigurationService.deleteScopedConfigurations(idsPinnedForV3) + } + } + + @Test + fun testProcessBreakingChangePinRollbacksWithNoBCsToRollBack() { + val breakingChanges = + listOf( + MockData.actorDefinitionBreakingChange("1.0.0"), + MockData.actorDefinitionBreakingChange("2.0.0"), + ) + + actorDefinitionVersionUpdater.processBreakingChangePinRollbacks(ACTOR_DEFINITION_ID, NEW_VERSION, breakingChanges) + + verify(exactly = 0) { + scopedConfigurationService.listScopedConfigurationsWithOrigins(any(), any(), any(), any(), any()) + scopedConfigurationService.deleteScopedConfigurations(any()) + } + } + + private fun buildBreakingChangeScopedConfig( + actorId: UUID, + breakingChange: ActorDefinitionBreakingChange, + ): ScopedConfiguration { + return ScopedConfiguration() + .withKey(ConnectorVersionKey.key) + .withValue(DEFAULT_VERSION.versionId.toString()) + .withResourceType(ConfigResourceType.ACTOR_DEFINITION) + .withResourceId(ACTOR_DEFINITION_ID) + .withScopeType(ConfigScopeType.ACTOR) + .withScopeId(actorId) + .withOriginType(ConfigOriginType.BREAKING_CHANGE) + .withOrigin(breakingChange.version.serialize()) + } + + private fun idsToConfigScopeMap(awoIds: ActorWorkspaceOrganizationIds): ConfigScopeMapWithId { + return ConfigScopeMapWithId( + awoIds.actorId, + mapOf( + ConfigScopeType.ACTOR to awoIds.actorId, + ConfigScopeType.WORKSPACE to awoIds.workspaceId, + ConfigScopeType.ORGANIZATION to awoIds.organizationId, + ), + ) + } } diff --git a/airbyte-data/src/test/kotlin/io/airbyte/data/repositories/ConnectionTimelineEventRepositoryTest.kt b/airbyte-data/src/test/kotlin/io/airbyte/data/repositories/ConnectionTimelineEventRepositoryTest.kt new file mode 100644 index 00000000000..0706e33ef3d --- /dev/null +++ b/airbyte-data/src/test/kotlin/io/airbyte/data/repositories/ConnectionTimelineEventRepositoryTest.kt @@ -0,0 +1,41 @@ +package io.airbyte.data.repositories + +import io.airbyte.data.repositories.entities.ConnectionTimelineEvent +import io.airbyte.db.instance.configs.jooq.generated.Keys +import io.airbyte.db.instance.configs.jooq.generated.Tables +import io.micronaut.test.extensions.junit5.annotation.MicronautTest +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import java.util.UUID + +@MicronautTest +internal class ConnectionTimelineEventRepositoryTest : AbstractConfigRepositoryTest( + ConnectionTimelineEventRepository::class, +) { + companion object { + @BeforeAll + @JvmStatic + fun setup() { + // so we don't have to deal with making users as well + jooqDslContext.alterTable( + Tables.CONNECTION_TIMELINE_EVENT, + ).dropForeignKey(Keys.CONNECTION_TIMELINE_EVENT__CONNECTION_TIMELINE_EVENT_CONNECTION_ID_FKEY.constraint()).execute() + } + } + + @Test + fun `test db insertion`() { + val eventId = java.util.UUID.randomUUID() + val event = + ConnectionTimelineEvent( + connectionId = UUID.randomUUID(), + eventType = "Test", + ) + + val saved = repository.save(event) + assert(repository.count() == 1L) + + val persistedEvent = repository.findById(saved.id!!).get() + assert(persistedEvent.connectionId == event.connectionId) + } +} diff --git a/airbyte-data/src/test/kotlin/io/airbyte/data/repositories/UserInvitationRepositoryTest.kt b/airbyte-data/src/test/kotlin/io/airbyte/data/repositories/UserInvitationRepositoryTest.kt index bed9c419b2f..aa35fc6c3b7 100644 --- a/airbyte-data/src/test/kotlin/io/airbyte/data/repositories/UserInvitationRepositoryTest.kt +++ b/airbyte-data/src/test/kotlin/io/airbyte/data/repositories/UserInvitationRepositoryTest.kt @@ -11,12 +11,15 @@ import io.micronaut.test.extensions.junit5.annotation.MicronautTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test +import java.time.OffsetDateTime +import java.time.ZoneOffset import java.util.UUID @MicronautTest internal class UserInvitationRepositoryTest : AbstractConfigRepositoryTest(UserInvitationRepository::class) { companion object { const val INVITE_CODE = "some-code" + val EXPIRES_AT = OffsetDateTime.now(ZoneOffset.UTC).plusDays(7).truncatedTo(java.time.temporal.ChronoUnit.SECONDS) val userInvitation = UserInvitation( @@ -27,6 +30,7 @@ internal class UserInvitationRepositoryTest : AbstractConfigRepositoryTest - val actual = actualWorkspaceInvitations.find { it.id == expected.id } + val actual = actualWorkspaceInvites.find { it.id == expected.id } assert(actual != null) assertInvitationEquals(expected, actual!!) } // for each organization invitation found, make sure that it has a match by calling assertInvitationEquals - expectedOrganizationMatches.forEach { expected -> - val actual = actualOrganizationInvitations.find { it.id == expected.id } + expectedOrgMatches.forEach { expected -> + val actual = actualOrgInvites.find { it.id == expected.id } + assert(actual != null) + assertInvitationEquals(expected, actual!!) + } + } + + @Test + fun `test find by status and scope type and scope id and invited email`() { + val workspaceId = UUID.randomUUID() + val otherWorkspaceId = UUID.randomUUID() + val matchingStatus = InvitationStatus.pending + val otherStatus = InvitationStatus.accepted + val matchingEmail = "matching@airbyte.io" + val otherEmail = "other@airbyte.io" + + val matchingInvite = + userInvitation.copy( + id = UUID.randomUUID(), + inviteCode = UUID.randomUUID().toString(), + scopeId = workspaceId, + status = matchingStatus, + invitedEmail = matchingEmail, + ) + repository.save(matchingInvite) + + val anotherMatchingInvite = + matchingInvite.copy( + id = UUID.randomUUID(), + inviteCode = UUID.randomUUID().toString(), + ) + repository.save(anotherMatchingInvite) + + val wrongEmailInvite = + matchingInvite.copy( + id = UUID.randomUUID(), + inviteCode = UUID.randomUUID().toString(), + invitedEmail = otherEmail, + ) + repository.save(wrongEmailInvite) + + val wrongWorkspaceInvite = + matchingInvite.copy( + id = UUID.randomUUID(), + inviteCode = UUID.randomUUID().toString(), + scopeId = otherWorkspaceId, + ) + repository.save(wrongWorkspaceInvite) + + val wrongStatusInvite = + matchingInvite.copy( + id = UUID.randomUUID(), + inviteCode = UUID.randomUUID().toString(), + status = otherStatus, + ) + repository.save(wrongStatusInvite) + + val wrongEverythingInvite = + userInvitation.copy( + id = UUID.randomUUID(), + inviteCode = UUID.randomUUID().toString(), + invitedEmail = otherEmail, + scopeId = otherWorkspaceId, + status = otherStatus, + ) + repository.save(wrongEverythingInvite) + + val expectedMatches = listOf(matchingInvite, anotherMatchingInvite) + val actualMatches = + repository.findByStatusAndScopeTypeAndScopeIdAndInvitedEmail( + matchingStatus, + EntityScopeType.workspace, + workspaceId, + matchingEmail, + ) + + // for each invitation found, make sure that it has a match by calling assertInvitationEquals + expectedMatches.forEach { expected -> + val actual = actualMatches.find { it.id == expected.id } assert(actual != null) assertInvitationEquals(expected, actual!!) } diff --git a/airbyte-data/src/test/kotlin/io/airbyte/data/services/impls/data/UserInvitationServiceDataImplTest.kt b/airbyte-data/src/test/kotlin/io/airbyte/data/services/impls/data/UserInvitationServiceDataImplTest.kt index a8955223685..6e006e32f2d 100644 --- a/airbyte-data/src/test/kotlin/io/airbyte/data/services/impls/data/UserInvitationServiceDataImplTest.kt +++ b/airbyte-data/src/test/kotlin/io/airbyte/data/services/impls/data/UserInvitationServiceDataImplTest.kt @@ -6,6 +6,8 @@ import io.airbyte.data.repositories.PermissionRepository import io.airbyte.data.repositories.UserInvitationRepository import io.airbyte.data.repositories.entities.Permission import io.airbyte.data.repositories.entities.UserInvitation +import io.airbyte.data.services.InvitationDuplicateException +import io.airbyte.data.services.InvitationStatusUnexpectedException import io.airbyte.data.services.impls.data.mappers.EntityInvitationStatus import io.airbyte.data.services.impls.data.mappers.EntityPermissionType import io.airbyte.data.services.impls.data.mappers.EntityScopeType @@ -42,6 +44,7 @@ internal class UserInvitationServiceDataImplTest { status = EntityInvitationStatus.pending, createdAt = OffsetDateTime.now(ZoneOffset.UTC).truncatedTo(java.time.temporal.ChronoUnit.SECONDS), updatedAt = OffsetDateTime.now(ZoneOffset.UTC).truncatedTo(java.time.temporal.ChronoUnit.SECONDS), + expiresAt = OffsetDateTime.now(ZoneOffset.UTC).plusDays(7).truncatedTo(java.time.temporal.ChronoUnit.SECONDS), ) @BeforeEach @@ -70,6 +73,7 @@ internal class UserInvitationServiceDataImplTest { @Test fun `test create user invitation`() { + every { userInvitationRepository.findByStatusAndScopeTypeAndScopeIdAndInvitedEmail(any(), any(), any(), any()) } returns emptyList() every { userInvitationRepository.save(invitation) } returns invitation val result = userInvitationService.createUserInvitation(invitation.toConfigModel()) @@ -78,10 +82,23 @@ internal class UserInvitationServiceDataImplTest { verify { userInvitationRepository.save(invitation) } } + @Test + fun `test create duplicate user invitation throws`() { + every { userInvitationRepository.findByStatusAndScopeTypeAndScopeIdAndInvitedEmail(any(), any(), any(), any()) } returns listOf(invitation) + + assertThrows { userInvitationService.createUserInvitation(invitation.toConfigModel()) } + + verify(exactly = 0) { userInvitationRepository.save(invitation) } + } + @Test fun `test accept user invitation`() { val invitedUserId = UUID.randomUUID() - val expectedUpdatedInvitation = invitation.copy(status = EntityInvitationStatus.accepted) + val expectedUpdatedInvitation = + invitation.copy( + status = EntityInvitationStatus.accepted, + acceptedByUserId = invitedUserId, + ) every { userInvitationRepository.findByInviteCode(invitation.inviteCode) } returns Optional.of(invitation) every { userInvitationRepository.update(expectedUpdatedInvitation) } returns expectedUpdatedInvitation @@ -117,11 +134,29 @@ internal class UserInvitationServiceDataImplTest { every { userInvitationRepository.findByInviteCode(invitation.inviteCode) } returns Optional.of(invitation) - assertThrows { userInvitationService.acceptUserInvitation(invitation.inviteCode, invitedUserId) } + assertThrows { userInvitationService.acceptUserInvitation(invitation.inviteCode, invitedUserId) } verify(exactly = 0) { userInvitationRepository.update(any()) } } + @Test + fun `test accept user invitation fails if expired`() { + val invitedUserId = UUID.randomUUID() + val expiredInvitation = + invitation.copy( + status = EntityInvitationStatus.pending, + expiresAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(1), + ) + val expectedUpdatedInvitation = expiredInvitation.copy(status = EntityInvitationStatus.expired) + + every { userInvitationRepository.findByInviteCode(expiredInvitation.inviteCode) } returns Optional.of(expiredInvitation) + every { userInvitationRepository.update(expectedUpdatedInvitation) } returns expectedUpdatedInvitation + + assertThrows { userInvitationService.acceptUserInvitation(expiredInvitation.inviteCode, invitedUserId) } + + verify { userInvitationRepository.update(expectedUpdatedInvitation) } + } + @Test fun `test get pending invitations`() { val workspaceId = UUID.randomUUID() @@ -150,4 +185,15 @@ internal class UserInvitationServiceDataImplTest { assert(workspaceResult == mockWorkspaceInvitations.map { it.toConfigModel() }) assert(organizationResult == mockOrganizationInvitations.map { it.toConfigModel() }) } + + @Test + fun `test cancel invitation`() { + val expectedUpdatedInvitation = invitation.copy(status = EntityInvitationStatus.cancelled) + every { userInvitationRepository.findByInviteCode(invitation.inviteCode) } returns Optional.of(invitation) + every { userInvitationRepository.update(expectedUpdatedInvitation) } returns expectedUpdatedInvitation + + userInvitationService.cancelUserInvitation(invitation.inviteCode) + + verify { userInvitationRepository.update(expectedUpdatedInvitation) } + } } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_41_010__AddConditionalMutexKeyIndexToWorkloads.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_41_010__AddConditionalMutexKeyIndexToWorkloads.java new file mode 100644 index 00000000000..0a0574532be --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_41_010__AddConditionalMutexKeyIndexToWorkloads.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import java.util.List; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// TODO: update migration description in the class name +public class V0_50_41_010__AddConditionalMutexKeyIndexToWorkloads extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_41_010__AddConditionalMutexKeyIndexToWorkloads.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + ctx.createIndexIfNotExists("active_workload_by_mutex_idx") + .on("workload", "mutex_key") + .where(DSL.field("status").in(List.of("pending", "claimed", "launched", "running"))) + .execute(); + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_41_011__AddUserInvitationAcceptedByAndExpiration.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_41_011__AddUserInvitationAcceptedByAndExpiration.java new file mode 100644 index 00000000000..004ab50f338 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_41_011__AddUserInvitationAcceptedByAndExpiration.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static io.airbyte.db.instance.DatabaseConstants.USER_INVITATION_TABLE; +import static io.airbyte.db.instance.DatabaseConstants.USER_TABLE; +import static org.jooq.impl.DSL.foreignKey; + +import java.sql.Timestamp; +import java.util.UUID; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Add accepted_by_user_id column and expires_at column to user_invitations table. Also add expired + * status to invitation_status enum. + */ +public class V0_50_41_011__AddUserInvitationAcceptedByAndExpiration extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_41_011__AddUserInvitationAcceptedByAndExpiration.class); + + private static final String ACCEPTED_BY_USER_ID = "accepted_by_user_id"; + private static final String EXPIRES_AT = "expires_at"; + private static final String INVITATION_STATUS = "invitation_status"; + private static final String EXPIRED = "expired"; + + private static final Field ACCEPTED_BY_USER_ID_COLUMN = DSL.field(ACCEPTED_BY_USER_ID, SQLDataType.UUID.nullable(true)); + private static final Field EXPIRES_AT_COLUMN = DSL.field(EXPIRES_AT, SQLDataType.TIMESTAMP.nullable(false)); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + final DSLContext ctx = DSL.using(context.getConnection()); + + addAcceptedByUserIdColumnAndIndex(ctx); + addExpiresAtColumnAndIndex(ctx); + addExpiredStatus(ctx); + } + + static void addAcceptedByUserIdColumnAndIndex(final DSLContext ctx) { + ctx.alterTable(USER_INVITATION_TABLE) + .addColumnIfNotExists(ACCEPTED_BY_USER_ID_COLUMN) + .execute(); + + ctx.alterTable(USER_INVITATION_TABLE) + .add(foreignKey(ACCEPTED_BY_USER_ID) + .references(USER_TABLE, "id") + .onDeleteCascade()) + .execute(); + + ctx.createIndex("user_invitation_accepted_by_user_id_index") + .on(USER_INVITATION_TABLE, ACCEPTED_BY_USER_ID) + .execute(); + } + + static void addExpiresAtColumnAndIndex(final DSLContext ctx) { + ctx.alterTable(USER_INVITATION_TABLE).addColumnIfNotExists(EXPIRES_AT_COLUMN).execute(); + + ctx.createIndex("user_invitation_expires_at_index") + .on(USER_INVITATION_TABLE, EXPIRES_AT) + .execute(); + } + + static void addExpiredStatus(final DSLContext ctx) { + ctx.alterType(INVITATION_STATUS).addValue(EXPIRED).execute(); + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_41_012__BreakingChangePinDataMigration.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_41_012__BreakingChangePinDataMigration.java new file mode 100644 index 00000000000..542f3773537 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_41_012__BreakingChangePinDataMigration.java @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.version.Version; +import io.airbyte.db.instance.configs.migrations.V0_50_33_014__AddScopedConfigurationTable.ConfigResourceType; +import io.airbyte.db.instance.configs.migrations.V0_50_33_014__AddScopedConfigurationTable.ConfigScopeType; +import io.airbyte.db.instance.configs.migrations.V0_50_41_009__AddBreakingChangeConfigOrigin.ConfigOriginType; +import jakarta.annotation.Nullable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class V0_50_41_012__BreakingChangePinDataMigration extends BaseJavaMigration { + + private static final String CONNECTOR_VERSION_KEY = "connector_version"; + private static final Table ACTOR = DSL.table("actor"); + private static final Table ACTOR_DEFINITION = DSL.table("actor_definition"); + private static final Table ACTOR_DEFINITION_BREAKING_CHANGE = DSL.table("actor_definition_breaking_change"); + private static final Table WORKSPACE = DSL.table("workspace"); + private static final Table SCOPED_CONFIGURATION = DSL.table("scoped_configuration"); + private static final Field ID = DSL.field("id", SQLDataType.UUID); + private static final Field KEY = DSL.field("key", SQLDataType.VARCHAR); + private static final Field RESOURCE_TYPE = DSL.field("resource_type", ConfigResourceType.class); + private static final Field RESOURCE_ID = DSL.field("resource_id", SQLDataType.UUID); + private static final Field SCOPE_TYPE = DSL.field("scope_type", ConfigScopeType.class); + private static final Field SCOPE_ID = DSL.field("scope_id", SQLDataType.UUID); + private static final Field VALUE = DSL.field("value", SQLDataType.VARCHAR); + private static final Field DESCRIPTION = DSL.field("description", SQLDataType.VARCHAR); + private static final Field ORIGIN_TYPE = DSL.field("origin_type", ConfigOriginType.class); + private static final Field ORIGIN = DSL.field("origin", SQLDataType.VARCHAR); + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_41_012__BreakingChangePinDataMigration.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + migrateBreakingChangePins(ctx); + } + + @VisibleForTesting + public void migrateBreakingChangePins(final DSLContext ctx) { + final List actorDefinitions = getActorDefinitions(ctx); + for (final ActorDefinition actorDefinition : actorDefinitions) { + migrateBreakingChangePinsForDefinition(ctx, actorDefinition); + } + } + + private void migrateBreakingChangePinsForDefinition(final DSLContext ctx, final ActorDefinition actorDefinition) { + final List unpinnedActorsNotOnDefaultVersion = getUnpinnedActorsNotOnDefaultVersion(ctx, actorDefinition); + final List breakingChangeVersions = getBreakingChangeVersionsForDefinition(ctx, actorDefinition.actorDefinitionId); + for (final Actor actor : unpinnedActorsNotOnDefaultVersion) { + final String originatingBreakingChange = getOriginatingBreakingChangeForVersion(ctx, actor.defaultVersionId, breakingChangeVersions); + createScopedConfiguration(ctx, actorDefinition.actorDefinitionId, originatingBreakingChange, ConfigScopeType.ACTOR, actor.actorId, + actor.defaultVersionId); + } + } + + private List getUnpinnedActorsNotOnDefaultVersion(final DSLContext ctx, final ActorDefinition actorDefinition) { + final List actors = getActorsNotOnDefaultVersion(ctx, actorDefinition); + final List actorIdsWithConfig = + getIdsWithConfig(ctx, actorDefinition.actorDefinitionId, ConfigScopeType.ACTOR, actors.stream().map(Actor::actorId).toList()); + final List workspaceIdsWithConfig = + getIdsWithConfig(ctx, actorDefinition.actorDefinitionId, ConfigScopeType.WORKSPACE, actors.stream().map(Actor::workspaceId).toList()); + final List orgIdsWithConfig = + getIdsWithConfig(ctx, actorDefinition.actorDefinitionId, ConfigScopeType.ORGANIZATION, actors.stream().map(Actor::organizationId).toList()); + + return actors.stream() + .filter(actor -> !actorIdsWithConfig.contains(actor.actorId())) + .filter(actor -> !workspaceIdsWithConfig.contains(actor.workspaceId())) + .filter(actor -> !orgIdsWithConfig.contains(actor.organizationId())) + .toList(); + } + + final Map versionBreakingChangeCache = new HashMap<>(); + + private String getOriginatingBreakingChangeForVersion(final DSLContext ctx, final UUID versionId, final List breakingChangeVersions) { + if (versionBreakingChangeCache.containsKey(versionId)) { + return versionBreakingChangeCache.get(versionId); + } + + final ActorDefinitionVersion version = getActorDefinitionVersion(ctx, versionId); + final Version pinnedVersion = new Version(version.dockerImageTag); + + final Optional breakingVersion = breakingChangeVersions.stream() + .filter(breakingChangeVersion -> breakingChangeVersion.greaterThan(pinnedVersion)) + .findFirst(); + + if (breakingVersion.isEmpty()) { + throw new IllegalStateException(String.format( + "Could not find a corresponding breaking change for pinned version %s on actor definition ID %s. " + + "Overriding actor versions without a breaking change is not supported.", + version.dockerImageTag, version.actorDefinitionId)); + } + + final String originatingBreakingChange = breakingVersion.get().serialize(); + versionBreakingChangeCache.put(versionId, originatingBreakingChange); + return originatingBreakingChange; + } + + private List getIdsWithConfig(final DSLContext ctx, + final UUID actorDefinitionId, + final ConfigScopeType scopeType, + final List scopeIds) { + return ctx.select(SCOPE_ID) + .from(SCOPED_CONFIGURATION) + .where(KEY.eq(CONNECTOR_VERSION_KEY)) + .and(RESOURCE_TYPE.eq(ConfigResourceType.ACTOR_DEFINITION)) + .and(RESOURCE_ID.eq(actorDefinitionId)) + .and(SCOPE_TYPE.eq(scopeType)) + .and(SCOPE_ID.in(scopeIds)) + .fetch() + .map(r -> r.get(SCOPE_ID)); + } + + private void createScopedConfiguration(final DSLContext ctx, + final UUID actorDefinitionId, + final String breakingChangeVersionTag, + final ConfigScopeType scopeType, + final UUID scopeId, + final UUID pinnedVersionId) { + ctx.insertInto(SCOPED_CONFIGURATION) + .columns(ID, KEY, RESOURCE_TYPE, RESOURCE_ID, SCOPE_TYPE, SCOPE_ID, ORIGIN_TYPE, ORIGIN, VALUE, DESCRIPTION) + .values( + UUID.randomUUID(), + CONNECTOR_VERSION_KEY, + ConfigResourceType.ACTOR_DEFINITION, actorDefinitionId, + scopeType, scopeId, + ConfigOriginType.BREAKING_CHANGE, breakingChangeVersionTag, + pinnedVersionId.toString(), + "Automated breaking change pin migration") + .execute(); + } + + private List getActorDefinitions(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID); + final Field defaultVersionId = DSL.field("default_version_id", SQLDataType.UUID); + + return ctx.select(id, defaultVersionId) + .from(ACTOR_DEFINITION) + .fetch() + .map(r -> new ActorDefinition(r.get(id), r.get(defaultVersionId))); + } + + private ActorDefinitionVersion getActorDefinitionVersion(final DSLContext ctx, final UUID versionId) { + final Field id = DSL.field("id", SQLDataType.UUID); + final Field actorDefinitionId = DSL.field("actor_definition_id", SQLDataType.UUID); + final Field dockerImageTag = DSL.field("docker_image_tag", SQLDataType.VARCHAR); + + return ctx.select(id, actorDefinitionId, dockerImageTag) + .from("actor_definition_version") + .where(id.eq(versionId)) + .fetchOne(r -> new ActorDefinitionVersion(r.get(id), r.get(actorDefinitionId), r.get(dockerImageTag))); + } + + private List getBreakingChangeVersionsForDefinition(final DSLContext ctx, final UUID actorDefinitionId) { + final Field actorDefinitionIdField = DSL.field("actor_definition_id", SQLDataType.UUID); + final Field version = DSL.field("version", SQLDataType.VARCHAR); + + return ctx.select(version) + .from(ACTOR_DEFINITION_BREAKING_CHANGE) + .where(actorDefinitionIdField.eq(actorDefinitionId)) + .fetch() + .map(r -> new Version(r.get(version))) + .stream().sorted(Version::versionCompareTo) + .toList(); + } + + private List getActorsNotOnDefaultVersion(final DSLContext ctx, final ActorDefinition actorDefinition) { + // Actor fields + final Field actorId = DSL.field("actor.id", SQLDataType.UUID); + final Field actorWorkspaceId = DSL.field("actor.workspace_id", SQLDataType.UUID); + final Field actorDefaultVersionId = DSL.field("actor.default_version_id", SQLDataType.UUID); + final Field actorDefinitionId = DSL.field("actor.actor_definition_id", SQLDataType.UUID); + + // Workspace fields + final Field workspaceId = DSL.field("workspace.id", SQLDataType.UUID); + final Field workspaceOrgId = DSL.field("workspace.organization_id", SQLDataType.UUID); + + return ctx.select(actorId, actorWorkspaceId, workspaceOrgId, actorDefaultVersionId) + .from(ACTOR) + .join(WORKSPACE).on(workspaceId.eq(actorWorkspaceId)) + .where(actorDefinitionId.eq(actorDefinition.actorDefinitionId)) + .and(actorDefaultVersionId.ne(actorDefinition.defaultVersionId)) + .fetch() + .map(r -> new Actor(r.get(actorId), r.get(actorWorkspaceId), r.get(workspaceOrgId), r.get(actorDefaultVersionId))); + + } + + record ActorDefinition(UUID actorDefinitionId, UUID defaultVersionId) {} + + record ActorDefinitionVersion(UUID versionId, UUID actorDefinitionId, String dockerImageTag) {} + + record Actor(UUID actorId, UUID workspaceId, @Nullable UUID organizationId, UUID defaultVersionId) {} + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_50_4_002__AddScopeStatusCreatedAtIndex.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_50_4_002__AddScopeStatusCreatedAtIndex.java index 223f14a9f09..cec6135cc20 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_50_4_002__AddScopeStatusCreatedAtIndex.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_50_4_002__AddScopeStatusCreatedAtIndex.java @@ -15,12 +15,14 @@ public class V0_50_4_002__AddScopeStatusCreatedAtIndex extends BaseJavaMigration private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_4_002__AddScopeStatusCreatedAtIndex.class); + static final String SCOPE_STATUS_CREATED_AT_INDEX_NAME = "scope_status_created_at_idx"; + @Override public void migrate(final Context context) throws Exception { LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); final DSLContext ctx = DSL.using(context.getConnection()); - ctx.query("CREATE INDEX CONCURRENTLY IF NOT EXISTS scope_status_created_at_idx ON jobs(scope, status, created_at DESC)").execute(); + ctx.query("CREATE INDEX CONCURRENTLY IF NOT EXISTS " + SCOPE_STATUS_CREATED_AT_INDEX_NAME + " ON jobs(scope, status, created_at DESC)").execute(); } // This prevents flyway from automatically wrapping the migration in a transaction. diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_50_4_003__AddScopeCreatedAtScopeNonTerminalIndexes.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_50_4_003__AddScopeCreatedAtScopeNonTerminalIndexes.java new file mode 100644 index 00000000000..f269b4bdd36 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_50_4_003__AddScopeCreatedAtScopeNonTerminalIndexes.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.jobs.migrations; + +import static io.airbyte.db.instance.jobs.migrations.V0_50_4_002__AddScopeStatusCreatedAtIndex.SCOPE_STATUS_CREATED_AT_INDEX_NAME; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class V0_50_4_003__AddScopeCreatedAtScopeNonTerminalIndexes extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_4_003__AddScopeCreatedAtScopeNonTerminalIndexes.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + final DSLContext ctx = DSL.using(context.getConnection()); + // helps with the general sorting of jobs by latest per connection + ctx.query("CREATE INDEX CONCURRENTLY IF NOT EXISTS scope_created_at_idx ON jobs(scope, created_at DESC)").execute(); + + // helps for looking for active jobs + ctx.query( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS scope_non_terminal_status_idx ON jobs(scope, status) " + + "WHERE status NOT IN ('failed', 'succeeded', 'cancelled')") + .execute(); + + // remove other index, as these two are more performant + ctx.query("DROP INDEX CONCURRENTLY " + SCOPE_STATUS_CREATED_AT_INDEX_NAME).execute(); + } + + // This prevents flyway from automatically wrapping the migration in a transaction. + // This is important because indexes cannot be created concurrently (i.e. without locking) from + // within a transaction. + @Override + public boolean canExecuteInTransaction() { + return false; + } + +} diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt index c0a35f9feb0..8271f303ed4 100644 --- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt @@ -357,6 +357,8 @@ create table "public"."user_invitation" ( "updated_at" timestamp(6) with time zone not null default current_timestamp, "scope_id" uuid not null, "scope_type" scope_type not null, + "accepted_by_user_id" uuid, + "expires_at" timestamp(6) not null, constraint "user_invitation_pkey" primary key ("id"), constraint "user_invitation_invite_code_key" unique ("invite_code") ); @@ -443,10 +445,14 @@ create index "sso_config_organization_id_idx" on "public"."sso_config"("organiza create index "connection_id_stream_name_namespace_idx" on "public"."stream_reset"("connection_id" asc, "stream_name" asc, "stream_namespace" asc); create index "user_auth_provider_auth_user_id_idx" on "public"."user"("auth_provider" asc, "auth_user_id" asc); create index "user_email_idx" on "public"."user"("email" asc); +create index "user_invitation_accepted_by_user_id_index" on "public"."user_invitation"("accepted_by_user_id" asc); +create index "user_invitation_expires_at_index" on "public"."user_invitation"("expires_at" asc); create index "user_invitation_invite_code_idx" on "public"."user_invitation"("invite_code" asc); create index "user_invitation_invited_email_idx" on "public"."user_invitation"("invited_email" asc); create index "user_invitation_scope_id_index" on "public"."user_invitation"("scope_id" asc); create index "user_invitation_scope_type_and_scope_id_index" on "public"."user_invitation"("scope_type" asc, "scope_id" asc); +create index "active_workload_by_mutex_idx" on "public"."workload"("mutex_key" asc) +where ((status = ANY (ARRAY['pending'::workload_status, 'claimed'::workload_status, 'launched'::workload_status, 'running'::workload_status]))); create index "workload_deadline_idx" on "public"."workload"("deadline" asc) where ((deadline IS NOT NULL)); create index "workload_mutex_idx" on "public"."workload"("mutex_key" asc); @@ -481,6 +487,7 @@ alter table "public"."schema_management" add constraint "schema_management_conne alter table "public"."sso_config" add constraint "sso_config_organization_id_fkey" foreign key ("organization_id") references "public"."organization" ("id"); alter table "public"."state" add constraint "state_connection_id_fkey" foreign key ("connection_id") references "public"."connection" ("id"); alter table "public"."user" add constraint "user_default_workspace_id_fkey" foreign key ("default_workspace_id") references "public"."workspace" ("id"); +alter table "public"."user_invitation" add constraint "user_invitation_accepted_by_user_id_fkey" foreign key ("accepted_by_user_id") references "public"."user" ("id"); alter table "public"."user_invitation" add constraint "user_invitation_inviter_user_id_fkey" foreign key ("inviter_user_id") references "public"."user" ("id"); alter table "public"."workload_label" add constraint "workload_label_workload_id_fkey" foreign key ("workload_id") references "public"."workload" ("id"); alter table "public"."workspace" add constraint "workspace_organization_id_fkey" foreign key ("organization_id") references "public"."organization" ("id"); diff --git a/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt index 347cdcccd56..b642c9dcce0 100644 --- a/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt @@ -127,7 +127,9 @@ create unique index "job_attempt_idx" on "public"."attempts"("job_id" asc, "atte create index "jobs_config_type_idx" on "public"."jobs"("config_type" asc); create index "jobs_scope_idx" on "public"."jobs"("scope" asc); create index "jobs_status_idx" on "public"."jobs"("status" asc); -create index "scope_status_created_at_idx" on "public"."jobs"("scope" asc, "status" asc, "created_at" desc); +create index "scope_created_at_idx" on "public"."jobs"("scope" asc, "created_at" desc); +create index "scope_non_terminal_status_idx" on "public"."jobs"("scope" asc, "status" asc) +where ((status <> ALL (ARRAY['failed'::job_status, 'succeeded'::job_status, 'cancelled'::job_status]))); create index "normalization_summary_attempt_id_idx" on "public"."normalization_summaries"("attempt_id" asc); create index "retry_state_connection_id_idx" on "public"."retry_states"("connection_id" asc); create index "retry_state_job_id_idx" on "public"."retry_states"("job_id" asc); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_41_012__BreakingChangePinDataMigrationTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_41_012__BreakingChangePinDataMigrationTest.java new file mode 100644 index 00000000000..a20526534c5 --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_41_012__BreakingChangePinDataMigrationTest.java @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; +import io.airbyte.db.instance.configs.migrations.V0_50_33_014__AddScopedConfigurationTable.ConfigResourceType; +import io.airbyte.db.instance.configs.migrations.V0_50_33_014__AddScopedConfigurationTable.ConfigScopeType; +import io.airbyte.db.instance.configs.migrations.V0_50_41_006__AlterSupportLevelAddArchived.SupportLevel; +import io.airbyte.db.instance.configs.migrations.V0_50_41_009__AddBreakingChangeConfigOrigin.ConfigOriginType; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import java.sql.Date; +import java.time.LocalDate; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.jooq.DSLContext; +import org.jooq.JSONB; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class V0_50_41_012__BreakingChangePinDataMigrationTest extends AbstractConfigsDatabaseTest { + + private V0_50_41_012__BreakingChangePinDataMigration migration; + + @BeforeEach + void beforeEach() { + final Flyway flyway = + FlywayFactory.create(dataSource, "V0_50_41_012__BreakingChangePinDataMigrationTest", ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final ConfigsDatabaseMigrator configsDbMigrator = new ConfigsDatabaseMigrator(database, flyway); + + final BaseJavaMigration previousMigration = new V0_50_41_009__AddBreakingChangeConfigOrigin(); + final DevDatabaseMigrator devConfigsDbMigrator = new DevDatabaseMigrator(configsDbMigrator, previousMigration.getVersion()); + devConfigsDbMigrator.createBaseline(); + + migration = new V0_50_41_012__BreakingChangePinDataMigration(); + } + + static Stream testMethodSource() { + return Stream.of( + // Already on latest (3.1.0), no BC pin + Arguments.of("3.1.0", List.of(), null), + + // Held back on an older version should create pin with correct BC as origin + Arguments.of("0.1.0", List.of(), "1.0.0"), + Arguments.of("1.0.0", List.of(), "2.0.0"), + + // Actors already pinned (at any level) should be ignored + Arguments.of("1.0.0", List.of(ConfigScopeType.ACTOR), null), + Arguments.of("1.0.0", List.of(ConfigScopeType.WORKSPACE), null), + Arguments.of("1.0.0", List.of(ConfigScopeType.ORGANIZATION), null), + Arguments.of("1.0.0", List.of(ConfigScopeType.ACTOR, ConfigScopeType.WORKSPACE, ConfigScopeType.ORGANIZATION), null)); + } + + @ParameterizedTest + @MethodSource("testMethodSource") + void testBreakingChangeOriginScopedConfig(final String actorVersion, + final List existingConfigScopes, + @Nullable final String expectedBCOrigin) { + final DSLContext ctx = getDslContext(); + + // ignore all foreign key constraints + ctx.execute("SET session_replication_role = replica;"); + + final UUID actorDefinitionId = UUID.randomUUID(); + createActorDefinition(ctx, actorDefinitionId); + + final UUID defaultVersionId = UUID.randomUUID(); + final String defaultVersionTag = "3.1.0"; + createActorDefinitionVersion(ctx, defaultVersionId, actorDefinitionId, defaultVersionTag); + setActorDefinitionDefaultVersion(ctx, actorDefinitionId, defaultVersionId); + + UUID actorVersionId = defaultVersionId; + if (!actorVersion.equals(defaultVersionTag)) { + actorVersionId = UUID.randomUUID(); + createActorDefinitionVersion(ctx, actorVersionId, actorDefinitionId, actorVersion); + } + + final UUID workspaceId = UUID.randomUUID(); + final UUID organizationId = UUID.randomUUID(); + createWorkspace(ctx, workspaceId, organizationId); + + final UUID actorId = UUID.randomUUID(); + createActor(ctx, actorId, workspaceId, actorDefinitionId, actorVersionId); + + for (final ConfigScopeType existingConfigScope : existingConfigScopes) { + final UUID scopeId; + switch (existingConfigScope) { + case ACTOR -> scopeId = actorId; + case WORKSPACE -> scopeId = workspaceId; + case ORGANIZATION -> scopeId = organizationId; + default -> throw new IllegalArgumentException("Unexpected config scope type: " + existingConfigScope); + } + createScopedConfig(ctx, actorDefinitionId, existingConfigScope, scopeId, ConfigOriginType.USER, "userId", actorVersion); + } + + final List breakingChanges = List.of("1.0.0", "2.0.0", "3.0.0"); + for (final String breakingChange : breakingChanges) { + createBreakingChange(ctx, actorDefinitionId, breakingChange); + } + + // run migration + migration.migrateBreakingChangePins(ctx); + + // get pin and assert it's correct + final Optional> scopedConfig = getScopedConfig(ctx, actorDefinitionId, actorId); + if (expectedBCOrigin == null) { + assert (scopedConfig.isEmpty()); + } else { + assert (scopedConfig.isPresent()); + assert (scopedConfig.get().get("value").equals(actorVersionId.toString())); + assert (scopedConfig.get().get("origin").equals(expectedBCOrigin)); + } + + } + + private static void createActorDefinition(final DSLContext ctx, final UUID actorDefinitionId) { + ctx.insertInto(DSL.table("actor_definition")) + .columns( + DSL.field("id"), + DSL.field("name"), + DSL.field("actor_type")) + .values( + actorDefinitionId, + "postgres", + ActorType.source) + .execute(); + } + + private static void setActorDefinitionDefaultVersion(final DSLContext ctx, final UUID actorDefinitionId, final UUID defaultVersionId) { + ctx.update(DSL.table("actor_definition")) + .set(DSL.field("default_version_id"), defaultVersionId) + .where(DSL.field("id").eq(actorDefinitionId)) + .execute(); + } + + private static void createActorDefinitionVersion(final DSLContext ctx, + final UUID actorDefinitionVersionId, + final UUID actorDefinitionId, + final String version) { + ctx.insertInto(DSL.table("actor_definition_version")) + .columns( + DSL.field("id"), + DSL.field("actor_definition_id"), + DSL.field("docker_repository"), + DSL.field("docker_image_tag"), + DSL.field("support_level"), + DSL.field("spec", SQLDataType.JSONB)) + .values( + actorDefinitionVersionId, + actorDefinitionId, + "airbyte/postgres", + version, + SupportLevel.community, + JSONB.valueOf("{}")) + .execute(); + } + + private static void createActor(final DSLContext ctx, + final UUID actorId, + final UUID workspaceId, + final UUID actorDefinitionId, + final UUID defaultVersionId) { + ctx.insertInto(DSL.table("actor")) + .columns( + DSL.field("id"), + DSL.field("name"), + DSL.field("actor_type"), + DSL.field("workspace_id"), + DSL.field("actor_definition_id"), + DSL.field("default_version_id"), + DSL.field("configuration", SQLDataType.JSONB)) + .values( + actorId, + "postgres", + ActorType.source, + workspaceId, + actorDefinitionId, + defaultVersionId, + JSONB.valueOf("{}")) + .execute(); + } + + private static void createWorkspace(final DSLContext ctx, final UUID workspaceId, final UUID organizationId) { + ctx.insertInto(DSL.table("workspace")) + .columns( + DSL.field("id"), + DSL.field("name"), + DSL.field("slug"), + DSL.field("initial_setup_complete"), + DSL.field("organization_id")) + .values( + workspaceId, + "workspace", + "workspace", + true, + organizationId) + .execute(); + } + + private static void createScopedConfig( + final DSLContext ctx, + final UUID actorDefinitionId, + final ConfigScopeType scopeType, + final UUID scopeId, + final ConfigOriginType originType, + final String origin, + final String value) { + ctx.insertInto(DSL.table("scoped_configuration")) + .columns( + DSL.field("id"), + DSL.field("key"), + DSL.field("resource_type"), + DSL.field("resource_id"), + DSL.field("scope_type"), + DSL.field("scope_id"), + DSL.field("value"), + DSL.field("origin_type"), + DSL.field("origin")) + .values( + UUID.randomUUID(), + "connector_version", + ConfigResourceType.ACTOR_DEFINITION, + actorDefinitionId, + scopeType, + scopeId, + value, + originType, + origin) + .execute(); + } + + private static Optional> getScopedConfig(final DSLContext ctx, final UUID actorDefinitionId, final UUID scopeId) { + return ctx.select(DSL.field("value"), DSL.field("origin")) + .from(DSL.table("scoped_configuration")) + .where(DSL.field("resource_type").eq(ConfigResourceType.ACTOR_DEFINITION) + .and(DSL.field("resource_id").eq(actorDefinitionId)) + .and(DSL.field("scope_type").eq(ConfigScopeType.ACTOR)) + .and(DSL.field("scope_id").eq(scopeId)) + .and(DSL.field("origin_type").eq(ConfigOriginType.BREAKING_CHANGE))) + .fetchOptional() + .map(r -> r + .map(record -> Map.of( + "value", record.get(DSL.field("value", String.class)), + "origin", record.get(DSL.field("origin", String.class))))); + } + + private static void createBreakingChange(final DSLContext ctx, final UUID actorDefinitionId, final String version) { + ctx.insertInto(DSL.table("actor_definition_breaking_change")) + .columns( + DSL.field("actor_definition_id"), + DSL.field("version"), + DSL.field("migration_documentation_url"), + DSL.field("message"), + DSL.field("upgrade_deadline", SQLDataType.DATE)) + .values( + actorDefinitionId, + version, + "https://docs.airbyte.io/", + "Breaking change", + Date.valueOf(LocalDate.now())) + .execute(); + } + +} diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index e4a598583a8..d3b45567ec0 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -111,6 +111,8 @@ object ConnectorApmEnabled : Permanent(key = "connectors.apm-enabled", object AutoRechargeEnabled : Permanent(key = "billing.autoRecharge", default = false) +object UseBreakingChangeScopedConfigs : Temporary(key = "connectors.useBreakingChangeScopedConfigs", default = false) + // NOTE: this is deprecated in favor of FieldSelectionEnabled and will be removed once that flag is fully deployed. object FieldSelectionWorkspaces : EnvVar(envVar = "FIELD_SELECTION_WORKSPACES") { override fun enabled(ctx: Context): Boolean { @@ -191,3 +193,5 @@ object BillingCronScopeChangeTimestamp : Permanent(key = "platform.billi object UseWorkloadApiForDiscover : Temporary(key = "platform.use-workload-api-for-discover", default = false) object UseWorkloadApiForSpec : Temporary(key = "platform.use-workload-api-for-spec", default = false) + +object EnforceMutexKeyOnCreate : Temporary(key = "platform.enforce-mutex-key-on-create", default = false) diff --git a/airbyte-keycloak-setup/Dockerfile b/airbyte-keycloak-setup/Dockerfile index 481e479b0be..ab40d89e5de 100644 --- a/airbyte-keycloak-setup/Dockerfile +++ b/airbyte-keycloak-setup/Dockerfile @@ -1,5 +1,11 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.0.1 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.1.0 FROM ${JDK_IMAGE} AS keycloak-setup + WORKDIR /app + +USER root ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte + ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/airbyte-keycloak-setup"] diff --git a/airbyte-keycloak-setup/build.gradle.kts b/airbyte-keycloak-setup/build.gradle.kts index eaba71edfbd..8a904297ee8 100644 --- a/airbyte-keycloak-setup/build.gradle.kts +++ b/airbyte-keycloak-setup/build.gradle.kts @@ -12,14 +12,17 @@ dependencies { annotationProcessor(platform(libs.micronaut.platform)) annotationProcessor(libs.bundles.micronaut.annotation.processor) - implementation( platform(libs.micronaut.platform)) - implementation( libs.bundles.micronaut) - implementation( libs.bundles.keycloak.client) + implementation(platform(libs.micronaut.platform)) + implementation(libs.bundles.micronaut) + implementation(libs.bundles.keycloak.client) implementation(project(":airbyte-commons")) implementation(project(":airbyte-commons-auth")) implementation(project(":airbyte-commons-micronaut")) implementation(project(":airbyte-commons-micronaut-security")) + implementation(project(":airbyte-data")) + implementation(project(":airbyte-db:db-lib")) + implementation(project(":airbyte-db:jooq")) testAnnotationProcessor(platform(libs.micronaut.platform)) testAnnotationProcessor(libs.bundles.micronaut.test.annotation.processor) @@ -27,6 +30,8 @@ dependencies { testImplementation(libs.bundles.micronaut.test) testImplementation(libs.bundles.junit) testImplementation(libs.junit.jupiter.system.stubs) + + testImplementation(project(":airbyte-test-utils")) } val env = Properties().apply { diff --git a/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/ConfigDbResetHelper.java b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/ConfigDbResetHelper.java new file mode 100644 index 00000000000..dc5316e8d37 --- /dev/null +++ b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/ConfigDbResetHelper.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.keycloak.setup; + +import io.airbyte.db.Database; +import io.airbyte.db.instance.configs.jooq.generated.Tables; +import io.airbyte.db.instance.configs.jooq.generated.enums.AuthProvider; +import jakarta.inject.Named; +import jakarta.inject.Singleton; +import java.sql.SQLException; + +/** + * Helper to reset the Config DB state as part of a Keycloak Realm Reset. Cleans up old User records + * that would otherwise become orphaned when the Keycloak Realm is recreated from scratch, and + * assigns new Keycloak auth IDs to SSO logins. + */ +@Singleton +public class ConfigDbResetHelper { + + private final Database configDb; + + public ConfigDbResetHelper(@Named("configDatabase") final Database configDb) { + this.configDb = configDb; + } + + public void deleteConfigDbUsers() throws SQLException { + // DO NOT REMOVE THIS CRITICAL CHECK. + throwIfMultipleOrganizations(); + + this.configDb.query(ctx -> ctx.deleteFrom(Tables.USER) + .where(Tables.USER.AUTH_PROVIDER.eq(AuthProvider.keycloak)) + .execute()); + } + + /** + * This reset operation would be detrimental if it runs in any sort of multi-organization instance. + * It relies on an assumption that all keycloak-backed users are part of the same + * organization/realm. If the one-and-only realm is reset, we know the users will be orphaned. This + * check is an extra layer of protection in case this code were somehow run in a multi-org + * environment like Airbyte Cloud or any future multi-org setup. + */ + private void throwIfMultipleOrganizations() throws SQLException { + final var orgCount = this.configDb.query(ctx -> ctx.fetchCount(Tables.ORGANIZATION)); + if (orgCount > 1) { + throw new IllegalStateException("Multiple organizations found in ConfigDb. " + + "This is not supported with the KEYCLOAK_RESET_REALM process."); + } + } + +} diff --git a/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/DatabaseBeanFactory.java b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/DatabaseBeanFactory.java new file mode 100644 index 00000000000..c39add63d7c --- /dev/null +++ b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/DatabaseBeanFactory.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.keycloak.setup; + +import io.airbyte.data.services.shared.DataSourceUnwrapper; +import io.airbyte.db.Database; +import io.micronaut.context.annotation.Factory; +import jakarta.inject.Named; +import jakarta.inject.Singleton; +import org.jooq.DSLContext; + +@Factory +public class DatabaseBeanFactory { + + @Singleton + @Named("configDatabase") + public Database configDatabase(@Named("config") final DSLContext dslContext) { + return new Database(DataSourceUnwrapper.unwrapContext(dslContext)); + } + +} diff --git a/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/KeycloakSetup.java b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/KeycloakSetup.java index aa3df086a2b..5ed20474323 100644 --- a/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/KeycloakSetup.java +++ b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/KeycloakSetup.java @@ -9,6 +9,7 @@ import io.micronaut.http.HttpResponse; import io.micronaut.http.client.HttpClient; import jakarta.inject.Singleton; +import java.sql.SQLException; import lombok.extern.slf4j.Slf4j; /** @@ -23,14 +24,17 @@ public class KeycloakSetup { private final HttpClient httpClient; private final KeycloakServer keycloakServer; private final AirbyteKeycloakConfiguration keycloakConfiguration; + private final ConfigDbResetHelper configDbResetHelper; public KeycloakSetup( final HttpClient httpClient, final KeycloakServer keycloakServer, - final AirbyteKeycloakConfiguration keycloakConfiguration) { + final AirbyteKeycloakConfiguration keycloakConfiguration, + final ConfigDbResetHelper configDbResetHelper) { this.httpClient = httpClient; this.keycloakServer = keycloakServer; this.keycloakConfiguration = keycloakConfiguration; + this.configDbResetHelper = configDbResetHelper; } public void run() { @@ -44,6 +48,15 @@ public void run() { if (keycloakConfiguration.getResetRealm()) { keycloakServer.recreateAirbyteRealm(); + log.info("Successfully recreated Airbyte Realm. Now deleting Airbyte User/Permission records..."); + try { + configDbResetHelper.deleteConfigDbUsers(); + } catch (SQLException e) { + log.error("Encountered an error while cleaning up Airbyte User/Permission records. " + + "You likely need to re-run this KEYCLOAK_RESET_REALM operation.", e); + throw new RuntimeException(e); + } + log.info("Successfully cleaned existing Airbyte User/Permission records. Reset finished successfully."); } else { keycloakServer.createAirbyteRealm(); } diff --git a/airbyte-keycloak-setup/src/main/resources/application.yml b/airbyte-keycloak-setup/src/main/resources/application.yml index 36588e8c90c..0f33cd8e334 100644 --- a/airbyte-keycloak-setup/src/main/resources/application.yml +++ b/airbyte-keycloak-setup/src/main/resources/application.yml @@ -18,3 +18,16 @@ airbyte: username: ${KEYCLOAK_ADMIN_USER:} password: ${KEYCLOAK_ADMIN_PASSWORD:} reset-realm: ${KEYCLOAK_RESET_REALM:false} + +datasources: + config: + connection-test-query: SELECT 1 + connection-timeout: 30000 + maximum-pool-size: 10 + minimum-idle: 0 + idle-timeout: 600000 + initialization-fail-timeout: -1 # Disable fail fast checking to avoid issues due to other pods not being started in time + url: ${DATABASE_URL} + driverClassName: org.postgresql.Driver + username: ${DATABASE_USER} + password: ${DATABASE_PASSWORD} diff --git a/airbyte-keycloak-setup/src/test/java/io/airbyte/keycloak/setup/ConfigDbResetHelperTest.java b/airbyte-keycloak-setup/src/test/java/io/airbyte/keycloak/setup/ConfigDbResetHelperTest.java new file mode 100644 index 00000000000..0e83fce350c --- /dev/null +++ b/airbyte-keycloak-setup/src/test/java/io/airbyte/keycloak/setup/ConfigDbResetHelperTest.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.keycloak.setup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import io.airbyte.db.instance.configs.jooq.generated.Tables; +import io.airbyte.db.instance.configs.jooq.generated.enums.AuthProvider; +import io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType; +import io.airbyte.test.utils.BaseConfigDatabaseTest; +import java.sql.SQLException; +import java.util.UUID; +import org.jooq.impl.TableImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class ConfigDbResetHelperTest extends BaseConfigDatabaseTest { + + private static final UUID KEYCLOAK_USER_1_ID = UUID.randomUUID(); + private static final UUID KEYCLOAK_USER_2_ID = UUID.randomUUID(); + private static final UUID NON_KEYCLOAK_USER_ID = UUID.randomUUID(); + private static final UUID ORGANIZATION_ID = UUID.randomUUID(); + + private ConfigDbResetHelper configDbResetHelper; + + @BeforeEach + void setUp() throws Exception { + configDbResetHelper = new ConfigDbResetHelper(database); + truncateAllTables(); + + // Pre-populate the database with test data + database.transaction(ctx -> { + ctx.insertInto(Tables.ORGANIZATION, Tables.ORGANIZATION.ID, Tables.ORGANIZATION.NAME, Tables.ORGANIZATION.EMAIL) + .values(ORGANIZATION_ID, "Org", "org@airbyte.io") + .execute(); + + // Insert sample users, some with AuthProvider as keycloak and one with a different AuthProvider + ctx.insertInto(Tables.USER, Tables.USER.ID, Tables.USER.AUTH_PROVIDER, Tables.USER.AUTH_USER_ID, Tables.USER.EMAIL, Tables.USER.NAME) + .values(KEYCLOAK_USER_1_ID, AuthProvider.keycloak, UUID.randomUUID().toString(), "one@airbyte.io", "User One") + .values(KEYCLOAK_USER_2_ID, AuthProvider.keycloak, UUID.randomUUID().toString(), "two@airbyte.io", "User Two") + .values(NON_KEYCLOAK_USER_ID, AuthProvider.airbyte, UUID.randomUUID().toString(), "three@airbyte.io", "User Three") + .execute(); + + // Insert permissions for these users + ctx.insertInto(Tables.PERMISSION, Tables.PERMISSION.ID, Tables.PERMISSION.USER_ID, Tables.PERMISSION.ORGANIZATION_ID, + Tables.PERMISSION.PERMISSION_TYPE) + .values(UUID.randomUUID(), KEYCLOAK_USER_1_ID, ORGANIZATION_ID, PermissionType.organization_admin) + .values(UUID.randomUUID(), KEYCLOAK_USER_2_ID, ORGANIZATION_ID, PermissionType.organization_member) + .values(UUID.randomUUID(), NON_KEYCLOAK_USER_ID, null, PermissionType.instance_admin) + .execute(); + + return null; + }); + } + + @Test + void throwsIfMultipleOrgsDetected() throws Exception { + // Insert a second organization + database.query(ctx -> { + ctx.insertInto(Tables.ORGANIZATION, Tables.ORGANIZATION.ID, Tables.ORGANIZATION.NAME, Tables.ORGANIZATION.EMAIL) + .values(UUID.randomUUID(), "Org 2", "org2@airbyte.io") + .execute(); + return null; + }); + + // Expect an exception to be thrown when the helper is invoked + assertThrows(IllegalStateException.class, () -> configDbResetHelper.deleteConfigDbUsers()); + + // Expect no records to be deleted + assertEquals(3, countRowsInTable(Tables.USER)); + assertEquals(3, countRowsInTable(Tables.PERMISSION)); + } + + @Test + void deleteConfigDbUsers_KeycloakUsersExist_UsersAndPermissionsDeleted() throws SQLException { + // Before deletion, assert the initial state of the database + assertEquals(3, countRowsInTable(Tables.USER)); + assertEquals(3, countRowsInTable(Tables.PERMISSION)); + + // Perform the deletion operation + configDbResetHelper.deleteConfigDbUsers(); + + // Assert the state of the database after deletion + // Expecting users with AuthProvider keycloak and their permissions to be deleted + assertEquals(1, countRowsInTable(Tables.USER)); + assertEquals(1, countRowsInTable(Tables.PERMISSION)); + + // Assert that the remaining user is the one not backed by keycloak + final var remainingUserAuthProvider = database.query(ctx -> ctx.select(Tables.USER.AUTH_PROVIDER) + .from(Tables.USER) + .fetchOne(Tables.USER.AUTH_PROVIDER)); + assertEquals(AuthProvider.airbyte, remainingUserAuthProvider); + + // Assert that the remaining permission is the one not associated with a keycloak user + final var remainingPermissionType = database.query(ctx -> ctx.select(Tables.PERMISSION.PERMISSION_TYPE) + .from(Tables.PERMISSION) + .fetchOne(Tables.PERMISSION.PERMISSION_TYPE)); + assertEquals(PermissionType.instance_admin, remainingPermissionType); + } + + private int countRowsInTable(final TableImpl table) throws SQLException { + return database.query(ctx -> ctx.selectCount().from(table).fetchOne(0, int.class)); + } + +} diff --git a/airbyte-keycloak-setup/src/test/java/io/airbyte/keycloak/setup/KeycloakSetupTest.java b/airbyte-keycloak-setup/src/test/java/io/airbyte/keycloak/setup/KeycloakSetupTest.java index b9e4c05fd42..ea75e680b04 100644 --- a/airbyte-keycloak-setup/src/test/java/io/airbyte/keycloak/setup/KeycloakSetupTest.java +++ b/airbyte-keycloak-setup/src/test/java/io/airbyte/keycloak/setup/KeycloakSetupTest.java @@ -33,6 +33,8 @@ class KeycloakSetupTest { private KeycloakServer keycloakServer; @Mock private AirbyteKeycloakConfiguration keycloakConfiguration; + @Mock + private ConfigDbResetHelper configDbResetHelper; private KeycloakSetup keycloakSetup; @@ -45,17 +47,18 @@ void setup() { when(blockingHttpClient.exchange(any(HttpRequest.class), eq(String.class))) .thenReturn(HttpResponse.ok()); - keycloakSetup = new KeycloakSetup(httpClient, keycloakServer, keycloakConfiguration); + keycloakSetup = new KeycloakSetup(httpClient, keycloakServer, keycloakConfiguration, configDbResetHelper); } @Test - void testRun() { + void testRun() throws Exception { keycloakSetup.run(); verify(httpClient).toBlocking(); verify(blockingHttpClient).exchange(any(HttpRequest.class), eq(String.class)); verify(keycloakServer).createAirbyteRealm(); verify(keycloakServer).closeKeycloakAdminClient(); + verify(configDbResetHelper, never()).deleteConfigDbUsers(); } @Test @@ -72,13 +75,14 @@ void testRunThrowsException() { } @Test - void testResetRealm() { + void testResetRealm() throws Exception { when(keycloakConfiguration.getResetRealm()).thenReturn(true); keycloakSetup.run(); verify(keycloakServer, times(0)).createAirbyteRealm(); verify(keycloakServer, times(1)).recreateAirbyteRealm(); + verify(configDbResetHelper, times(1)).deleteConfigDbUsers(); } } diff --git a/airbyte-keycloak/themes/airbyte-cloud/email/messages/messages_en.properties b/airbyte-keycloak/themes/airbyte-cloud/email/messages/messages_en.properties new file mode 100644 index 00000000000..82eba373e5d --- /dev/null +++ b/airbyte-keycloak/themes/airbyte-cloud/email/messages/messages_en.properties @@ -0,0 +1,2 @@ +executeActionsSubject=Verify your email for Airbyte +executeActionsBodyHtml=

Hello,

Follow this link to verify your email address.

{0}

This link will expire within {4}.

If you didn’t ask to verify this address, you can ignore this email.

Thanks,

Your Airbyte team

diff --git a/airbyte-keycloak/themes/airbyte-cloud/email/theme.properties b/airbyte-keycloak/themes/airbyte-cloud/email/theme.properties new file mode 100644 index 00000000000..f1dbb7215d4 --- /dev/null +++ b/airbyte-keycloak/themes/airbyte-cloud/email/theme.properties @@ -0,0 +1 @@ +parent=base \ No newline at end of file diff --git a/airbyte-keycloak/themes/airbyte-cloud/login/login.ftl b/airbyte-keycloak/themes/airbyte-cloud/login/login.ftl index 718951fad2c..73ad4174f8c 100644 --- a/airbyte-keycloak/themes/airbyte-cloud/login/login.ftl +++ b/airbyte-keycloak/themes/airbyte-cloud/login/login.ftl @@ -28,7 +28,7 @@
- - - -
- ); -}; - -/** - * Returns a function that can be used to open a confirmation modal for deleting a - * workspace. The user must type the workspace name in a confirmation input in order to - * proceed with the deletion. - * - * @param workspace - the workspace to delete - * @param deleteWorkspace - the API function which will actually delete the workspace upon successful confirmation - */ -export const useConfirmWorkspaceDeletionModal = ( - workspace: WorkspaceRead, - deleteWorkspace: UseMutateAsyncFunction -) => { - const { formatMessage } = useIntl(); - const { registerNotification } = useNotificationService(); - const navigate = useNavigate(); - const { openModal } = useModalService(); - - return async () => { - const result = await openModal<"confirm">({ - title: formatMessage( - { - id: "settings.workspaceSettings.deleteWorkspace.confirmation.title", - }, - { name: workspace.name } - ), - content: ({ onClose }) => ( - onClose("confirm")} /> - ), - size: "md", - }); - - // type "closed" and reason "confirm" indicates a successful confirmation; "canceled" [sic] is its counterpart - // when the user backs out - if (result.type === "closed" && result.reason === "confirm") { - try { - await deleteWorkspace(workspace.workspaceId); - registerNotification({ - id: "settings.workspace.delete.success", - text: formatMessage({ id: "settings.workspaceSettings.delete.success" }), - type: "success", - }); - navigate(`/${RoutePaths.Workspaces}`); - } catch { - registerNotification({ - id: "settings.workspace.delete.error", - text: formatMessage({ id: "settings.workspaceSettings.delete.error" }), - type: "error", - }); - } - } - }; -}; diff --git a/airbyte-webapp/src/components/ConnectorBuilderProjectTable/ConnectorBuilderProjectTable.tsx b/airbyte-webapp/src/components/ConnectorBuilderProjectTable/ConnectorBuilderProjectTable.tsx index 5f8d0d09042..79e4514c5a0 100644 --- a/airbyte-webapp/src/components/ConnectorBuilderProjectTable/ConnectorBuilderProjectTable.tsx +++ b/airbyte-webapp/src/components/ConnectorBuilderProjectTable/ConnectorBuilderProjectTable.tsx @@ -1,13 +1,13 @@ import { createColumnHelper } from "@tanstack/react-table"; import classNames from "classnames"; import { useCallback, useMemo, useState } from "react"; -import { FormattedMessage } from "react-intl"; +import { FormattedMessage, useIntl } from "react-intl"; import { Button } from "components/ui/Button"; import { FlexContainer, FlexItem } from "components/ui/Flex"; import { Icon } from "components/ui/Icon"; import { Link } from "components/ui/Link"; -import { Modal, ModalBody } from "components/ui/Modal"; +import { ModalBody } from "components/ui/Modal"; import { Spinner } from "components/ui/Spinner"; import { Table } from "components/ui/Table"; import { Text } from "components/ui/Text"; @@ -23,6 +23,7 @@ import { import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useIntent } from "core/utils/rbac"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; +import { useModalService } from "hooks/services/Modal"; import { useNotificationService } from "hooks/services/Notification"; import { getEditPath } from "pages/connectorBuilder/ConnectorBuilderRoutes"; @@ -39,8 +40,8 @@ const CHANGED_VERSION_ERROR_ID = "connectorBuilder.changeVersion.error"; const VersionChangeModal: React.FC<{ project: BuilderProject; - onClose: () => void; -}> = ({ onClose, project }) => { + onComplete: () => void; +}> = ({ onComplete, project }) => { const { data: versions, isLoading: isLoadingVersionList } = useListBuilderProjectVersions(project); const [selectedVersion, setSelectedVersion] = useState(undefined); const { mutateAsync: changeVersion, isLoading } = useChangeBuilderProjectVersion(); @@ -74,7 +75,7 @@ const VersionChangeModal: React.FC<{ ), type: "success", }); - onClose(); + onComplete(); } catch (e) { registerNotification({ id: NOTIFICATION_ID, @@ -93,53 +94,49 @@ const VersionChangeModal: React.FC<{ } return ( - } - onClose={onClose} - > - - {isLoadingVersionList ? ( - - - - ) : ( - - {(versions || []).map((version, index) => ( - - ))} - - )} - - + + {version.isActive && ( + + + + )} + {isLoading && version.version === selectedVersion && } + + + ))} + + )} + ); }; const VersionChanger = ({ project, canUpdateConnector }: { project: BuilderProject; canUpdateConnector: boolean }) => { - const [changeInProgress, setChangeInProgress] = useState(false); + const { openModal } = useModalService(); + const { formatMessage } = useIntl(); + if (project.version === "draft") { return ( ); } + + const openVersionChangeModal = () => + openModal({ + title: formatMessage({ id: "connectorBuilder.changeVersionModal.title" }, { name: project.name }), + size: "sm", + content: ({ onComplete }) => , + }); + return ( - <> - - {changeInProgress && setChangeInProgress(false)} project={project} />} - + ); }; @@ -302,6 +302,7 @@ export const ConnectorBuilderProjectTable = ({ data={projects} className={styles.table} sorting={false} + stickyHeaders={false} initialSortBy={[{ id: "name", desc: false }]} /> ); diff --git a/airbyte-webapp/src/components/InitialBadge/InitialBadge.module.scss b/airbyte-webapp/src/components/InitialBadge/InitialBadge.module.scss index 5df450252b8..1fc97d5e36c 100644 --- a/airbyte-webapp/src/components/InitialBadge/InitialBadge.module.scss +++ b/airbyte-webapp/src/components/InitialBadge/InitialBadge.module.scss @@ -48,4 +48,5 @@ width: 24px; height: 24px; border-radius: variables.$border-radius-xs; + aspect-ratio: 1 / 1; } diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx index 56e2e74d5f2..54569a92264 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx +++ b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx @@ -185,7 +185,7 @@ export const NotificationSettingsForm: React.FC = () => { )} - + ); diff --git a/airbyte-webapp/src/components/WorkspaceEmailForm/WorkspaceEmailForm.tsx b/airbyte-webapp/src/components/WorkspaceEmailForm/WorkspaceEmailForm.tsx index b9f0798e134..01de92ea1ac 100644 --- a/airbyte-webapp/src/components/WorkspaceEmailForm/WorkspaceEmailForm.tsx +++ b/airbyte-webapp/src/components/WorkspaceEmailForm/WorkspaceEmailForm.tsx @@ -68,14 +68,13 @@ export const WorkspaceEmailForm = () => { name="email" fieldType="input" - inline - description={formatMessage({ id: "settings.notifications.emailRecipient" })} + labelTooltip={formatMessage({ id: "settings.notifications.emailRecipient" })} label={formatMessage({ id: "settings.workspaceSettings.updateWorkspaceNameForm.email.label" })} placeholder={formatMessage({ id: "settings.workspaceSettings.updateWorkspaceNameForm.email.placeholder", })} /> - + ); }; diff --git a/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx b/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx index 6234f23733b..836d6c06dbb 100644 --- a/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx +++ b/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx @@ -5,7 +5,7 @@ import { NavigateFunction, useNavigate } from "react-router-dom"; import { useLocation } from "react-use"; import { LocationSensorState } from "react-use/lib/useLocation"; -import { CommonRequestError, isVersionError } from "core/api"; +import { CommonRequestError } from "core/api"; import { isFormBuildError } from "core/form/FormBuildError"; import { trackError } from "core/utils/datadog"; import { TrackErrorFn } from "hooks/services/AppMonitoringService"; @@ -23,7 +23,6 @@ interface ApiErrorBoundaryState { } enum ErrorId { - VersionMismatch = "version.mismatch", FormBuild = "form.build", ServerUnavailable = "server.unavailable", UnknownError = "unknown", @@ -51,11 +50,6 @@ class ApiErrorBoundaryComponent extends React.Component< }; static getDerivedStateFromError(error: { message: string; status?: number; __type?: string }): ApiErrorBoundaryState { - // Update state so the next render will show the fallback UI. - if (isVersionError(error)) { - return { errorId: ErrorId.VersionMismatch, message: error.message }; - } - if (isFormBuildError(error)) { return { errorId: ErrorId.FormBuild, message: error.message }; } @@ -103,10 +97,6 @@ class ApiErrorBoundaryComponent extends React.Component< const { navigate, children } = this.props; const { errorId, didRetry, message, retryDelay } = this.state; - if (errorId === ErrorId.VersionMismatch) { - return ; - } - if (errorId === ErrorId.FormBuild) { return ( void; - title: string; - text: string; + title: string | React.ReactNode; + text: string | React.ReactNode; textValues?: Record; - submitButtonText: string; + onCancel: () => void; onSubmit: () => void; - submitButtonDataId?: string; cancelButtonText?: string; + confirmationText?: string; + submitButtonText: string; + submitButtonDataId?: string; additionalContent?: React.ReactNode; submitButtonVariant?: "danger" | "primary"; } export const ConfirmationModal: React.FC = ({ - onClose, title, text, additionalContent, textValues, + onCancel, onSubmit, submitButtonText, submitButtonDataId, cancelButtonText, + confirmationText, submitButtonVariant = "danger", }) => { const { isLoading, startAction } = useLoadingState(); const onSubmitBtnClick = () => startAction({ action: async () => onSubmit() }); + const [confirmationValue, setConfirmationValue] = React.useState(""); return ( - } testId="confirmationModal"> + : title} + testId="confirmationModal" + >
- + {isString(text) ? : text} {additionalContent} + {confirmationText && ( + + + {/* eslint-disable-next-line jsx-a11y/label-has-associated-control -- eslint loses the input even though it has an "htmlFor" */} + + setConfirmationValue(event.target.value)} + value={confirmationValue} + /> + + + )}
diff --git a/airbyte-webapp/src/components/common/ConnectionDeleteBlock/index.ts b/airbyte-webapp/src/components/common/ConnectionDeleteBlock/index.ts new file mode 100644 index 00000000000..eb0cf7da406 --- /dev/null +++ b/airbyte-webapp/src/components/common/ConnectionDeleteBlock/index.ts @@ -0,0 +1 @@ +export * from "./ConnectionDeleteBlock"; diff --git a/airbyte-webapp/src/components/common/DeleteBlock/index.ts b/airbyte-webapp/src/components/common/DeleteBlock/index.ts deleted file mode 100644 index efdf28e28c2..00000000000 --- a/airbyte-webapp/src/components/common/DeleteBlock/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./DeleteBlock"; diff --git a/airbyte-webapp/src/components/common/Version/Version.tsx b/airbyte-webapp/src/components/common/Version/Version.tsx index 98383640430..c5f86678e7f 100644 --- a/airbyte-webapp/src/components/common/Version/Version.tsx +++ b/airbyte-webapp/src/components/common/Version/Version.tsx @@ -2,10 +2,9 @@ import React from "react"; import { Text } from "components/ui/Text"; -import { useConfig } from "core/config"; +import { config } from "core/config"; export const Version: React.FC = () => { - const config = useConfig(); return ( {config.version} diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.stories.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.stories.tsx index d613bf9dd9c..b20108c4d3d 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.stories.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.stories.tsx @@ -1,4 +1,4 @@ -import { ComponentStory, ComponentMeta } from "@storybook/react"; +import { ComponentMeta, ComponentStory } from "@storybook/react"; import { FormattedMessage } from "react-intl"; import { Modal } from "components/ui/Modal"; @@ -17,7 +17,7 @@ const Template: ComponentStory = (args) => { return ( }> - null} /> + null} /> ); diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.test.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.test.tsx index 68cbd0aa4da..6c6d298a97f 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.test.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.test.tsx @@ -157,7 +157,7 @@ describe("catalog diff modal", () => { { + onComplete={() => { return null; }} /> @@ -206,7 +206,7 @@ describe("catalog diff modal", () => { { + onComplete={() => { return null; }} /> @@ -227,7 +227,7 @@ describe("catalog diff modal", () => { { + onComplete={() => { return null; }} /> @@ -248,7 +248,7 @@ describe("catalog diff modal", () => { { + onComplete={() => { return null; }} /> diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx index f6c29bc3cd5..405805e6701 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx @@ -14,10 +14,10 @@ import { getSortedDiff } from "./utils"; interface CatalogDiffModalProps { catalogDiff: CatalogDiff; catalog: AirbyteCatalog; - onClose: () => void; + onComplete: () => void; } -export const CatalogDiffModal: React.FC = ({ catalogDiff, catalog, onClose }) => { +export const CatalogDiffModal: React.FC = ({ catalogDiff, catalog, onComplete }) => { const { newItems, removedItems, changedItems } = useMemo( () => getSortedDiff(catalogDiff.transforms), [catalogDiff.transforms] @@ -33,7 +33,7 @@ export const CatalogDiffModal: React.FC = ({ catalogDiff,
- diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/CustomTransformationsFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/CustomTransformationsFormField.tsx index 5c6ee27ca94..bcdc1845136 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/CustomTransformationsFormField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/CustomTransformationsFormField.tsx @@ -17,7 +17,7 @@ export const CustomTransformationsFormField: React.FC = () => { const { fields, append, remove, update, move } = useFieldArray({ name: "transformations", }); - const { openModal, closeModal } = useModalService(); + const { openModal } = useModalService(); const defaultTransformation: OperationCreate = useMemo( () => ({ @@ -36,10 +36,10 @@ export const CustomTransformationsFormField: React.FC = () => { ); const openEditModal = (transformationItemIndex?: number) => - openModal({ + openModal({ size: "xl", title: , - content: () => ( + content: ({ onComplete, onCancel }) => ( { isDefined(transformationItemIndex) ? update(transformationItemIndex, transformation) : append(transformation); - closeModal(); + onComplete(); }} - onCancel={closeModal} + onCancel={onCancel} /> ), }); diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/DestinationStreamPrefixNameFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/DestinationStreamPrefixNameFormField.tsx index d98ba2bbbe6..be5fc0eb277 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/DestinationStreamPrefixNameFormField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/DestinationStreamPrefixNameFormField.tsx @@ -21,7 +21,7 @@ import { export const DestinationStreamPrefixNameFormField = () => { const { formatMessage } = useIntl(); const { setValue, control } = useFormContext(); - const { openModal, closeModal } = useModalService(); + const { openModal } = useModalService(); const prefix = useWatch({ name: "prefix", control }); const destinationStreamNamesChange = useCallback( @@ -39,20 +39,23 @@ export const DestinationStreamPrefixNameFormField = () => { const openDestinationStreamNamesModal = useCallback( () => - openModal({ + openModal({ size: "sm", title: , - content: () => ( + content: ({ onComplete, onCancel }) => ( { + destinationStreamNamesChange(values); + onComplete(); + }} /> ), }), - [closeModal, destinationStreamNamesChange, openModal, prefix] + [destinationStreamNamesChange, openModal, prefix] ); return ( diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionFormField.tsx index 92c54a04daa..ec5d84c66a2 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionFormField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionFormField.tsx @@ -14,11 +14,11 @@ import { useModalService } from "hooks/services/Modal"; import { FormConnectionFormValues } from "./formConfig"; import { FormFieldLayout } from "./FormFieldLayout"; import { namespaceDefinitionOptions } from "./types"; -import { DestinationNamespaceModal, DestinationNamespaceFormValues } from "../DestinationNamespaceModal"; +import { DestinationNamespaceFormValues, DestinationNamespaceModal } from "../DestinationNamespaceModal"; export const NamespaceDefinitionFormField = () => { const { setValue, control } = useFormContext(); - const { openModal, closeModal } = useModalService(); + const { openModal } = useModalService(); const namespaceDefinition = useWatch({ name: "namespaceDefinition", control }); const namespaceFormat = useWatch({ name: "namespaceFormat", control }); @@ -38,21 +38,24 @@ export const NamespaceDefinitionFormField = () => { const openDestinationNamespaceModal = useCallback( () => - openModal({ + openModal({ size: "lg", title: , - content: () => ( + content: ({ onComplete, onCancel }) => ( { + destinationNamespaceChange(values); + onComplete(); + }} /> ), }), - [closeModal, destinationNamespaceChange, namespaceDefinition, namespaceFormat, openModal] + [destinationNamespaceChange, namespaceDefinition, namespaceFormat, openModal] ); return ( diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogCard.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogCard.tsx index 15b0d5551fa..90fbd9e120c 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogCard.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/SyncCatalogCard.tsx @@ -15,6 +15,7 @@ import { LoadingBackdrop } from "components/ui/LoadingBackdrop"; import { naturalComparatorBy } from "core/utils/objects"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; +import { useExperiment } from "hooks/services/Experiment"; import { FormConnectionFormValues, SyncStreamFieldWithId } from "./formConfig"; import { useRefreshSourceSchemaWithConfirmationOnDirty } from "./refreshSourceSchemaWithConfirmationOnDirty"; @@ -45,6 +46,7 @@ export const SyncCatalogCard: React.FC = () => { name: "syncCatalog.streams", control, }); + const isSimplifiedCreation = useExperiment("connection.simplifiedCreation", false); const watchedPrefix = useWatch({ name: "prefix", control }); const watchedNamespaceDefinition = useWatch({ name: "namespaceDefinition", control }); @@ -86,12 +88,17 @@ export const SyncCatalogCard: React.FC = () => { }; }, [locationState?.action, locationState?.namespace, locationState?.streamName, filteredStreams]); + let cardTitle = mode === "readonly" ? "form.dataSync.readonly" : "form.dataSync"; + if (isSimplifiedCreation) { + cardTitle = mode === "readonly" ? "connectionForm.selectStreams.readonly" : "connectionForm.selectStreams"; + } + return ( - + {mode !== "readonly" && (
diff --git a/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx b/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx index dbf07bc112a..ea044bb31e6 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx @@ -13,6 +13,7 @@ import { Link } from "components/ui/Link"; import { useCurrentWorkspaceLink } from "area/workspace/utils"; import { useConnectionList, useDestinationList } from "core/api"; +import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { useExperiment } from "hooks/services/Experiment"; import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; @@ -29,6 +30,7 @@ export const DESTINATION_TYPE_PARAM = "destinationType"; export const DESTINATION_ID_PARAM = "destinationId"; export const SelectDestination: React.FC = () => { + useTrackPage(PageTrackingCodes.CONNECTIONS_NEW_DEFINE_DESTINATION); const { destinations } = useDestinationList(); const connectionList = useConnectionList(); const [searchParams, setSearchParams] = useSearchParams(); diff --git a/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx b/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx index b508558473d..16295225f08 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx @@ -10,6 +10,7 @@ import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { useSourceDefinitionList, useSourceList } from "core/api"; +import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { CreateNewSource, SOURCE_DEFINITION_PARAM } from "./CreateNewSource"; import { RadioButtonTiles } from "./RadioButtonTiles"; @@ -23,6 +24,7 @@ export const SOURCE_TYPE_PARAM = "sourceType"; export const SOURCE_ID_PARAM = "sourceId"; export const SelectSource: React.FC = () => { + useTrackPage(PageTrackingCodes.CONNECTIONS_NEW_DEFINE_SOURCE); const { sources } = useSourceList(); const { sourceDefinitionMap } = useSourceDefinitionList(); const [searchParams, setSearchParams] = useSearchParams(); diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx index 8d29cd597af..f349964d032 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx @@ -17,7 +17,6 @@ import { import { mockTheme } from "test-utils/mock-data/mockTheme"; import { mocked, TestWrapper, useMockIntersectionObserver } from "test-utils/testutils"; -import type { SchemaError } from "core/api"; import { useDiscoverSchema } from "core/api"; import { defaultOssFeatures, FeatureItem } from "core/services/features"; @@ -46,7 +45,7 @@ jest.mock("core/api", () => ({ useSourceDefinition: () => mockSourceDefinition, useDestinationDefinition: () => mockDestinationDefinition, useDiscoverSchema: jest.fn(() => mockBaseUseDiscoverSchema), - LogsRequestError: jest.requireActual("core/api/errors").LogsRequestError, + ErrorWithJobInfo: jest.requireActual("core/api/errors").ErrorWithJobInfo, })); jest.mock("area/connector/utils", () => ({ @@ -101,7 +100,7 @@ describe("CreateConnectionForm", () => { it("should render with an error", async () => { mocked(useDiscoverSchema).mockImplementationOnce(() => ({ ...mockBaseUseDiscoverSchema, - schemaErrorStatus: new Error("Test Error") as SchemaError, + schemaErrorStatus: new Error("Test Error"), })); const renderResult = await render(); diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx index 0c1ca7ea8ce..eb2c4eaa93b 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx @@ -5,7 +5,7 @@ import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; -import { LogsRequestError, SchemaError as SchemaErrorType } from "core/api"; +import { ErrorWithJobInfo } from "core/api"; import styles from "./SchemaError.module.scss"; @@ -13,10 +13,10 @@ export const SchemaError = ({ schemaError, refreshSchema, }: { - schemaError: Exclude; + schemaError: Error; refreshSchema: () => Promise; }) => { - const job = LogsRequestError.extractJobInfo(schemaError); + const job = ErrorWithJobInfo.getJobInfo(schemaError); return ( diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/ConnectorNamespaceConfiguration.ts b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/ConnectorNamespaceConfiguration.ts new file mode 100644 index 00000000000..6f08f94637c --- /dev/null +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/ConnectorNamespaceConfiguration.ts @@ -0,0 +1,19 @@ +import { ConnectorIds } from "area/connector/utils"; + +export const SourceNamespaceConfiguration = { + [ConnectorIds.Sources.E2ETesting]: { supportsNamespaces: false }, + [ConnectorIds.Sources.EndToEndTesting]: { supportsNamespaces: false }, +} as const; + +export const DestinationNamespaceConfiguration = { + [ConnectorIds.Destinations.BigQuery]: { supportsNamespaces: true, defaultNamespacePath: "dataset_id" }, + [ConnectorIds.Destinations.E2ETesting]: { supportsNamespaces: false }, + [ConnectorIds.Destinations.EndToEndTesting]: { supportsNamespaces: false }, + [ConnectorIds.Destinations.Milvus]: { supportsNamespaces: true }, + [ConnectorIds.Destinations.Pinecone]: { supportsNamespaces: true }, + [ConnectorIds.Destinations.Postgres]: { supportsNamespaces: true, defaultNamespacePath: "schema" }, + [ConnectorIds.Destinations.Redshift]: { supportsNamespaces: true, defaultNamespacePath: "schema" }, + [ConnectorIds.Destinations.S3]: { supportsNamespaces: true }, + [ConnectorIds.Destinations.Snowflake]: { supportsNamespaces: true, defaultNamespacePath: "schema" }, + [ConnectorIds.Destinations.Weaviate]: { supportsNamespaces: false }, +} as const; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx index f74a5868a57..27a174a0926 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx @@ -19,6 +19,7 @@ import { Text } from "components/ui/Text"; import { useGetDestinationFromSearchParams, useGetSourceFromSearchParams } from "area/connector/utils"; import { useCurrentWorkspaceLink } from "area/workspace/utils"; +import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; @@ -54,6 +55,7 @@ export const SimplifiedConnectionConfiguration: React.FC = () => { }; const SimplifiedConnectionCreationReplication: React.FC = () => { + useTrackPage(PageTrackingCodes.CONNECTIONS_NEW_SELECT_STREAMS); const { formatMessage } = useIntl(); const { isDirty } = useFormState(); const { trackFormChange } = useFormChangeTrackerService(); @@ -74,11 +76,13 @@ const SimplifiedConnectionCreationReplication: React.FC = () => { }; const SimplifiedConnectionCreationConfigureConnection: React.FC = () => { + useTrackPage(PageTrackingCodes.CONNECTIONS_NEW_CONFIGURE_CONNECTION); const { formatMessage } = useIntl(); const { isDirty } = useFormState(); const { trackFormChange } = useFormChangeTrackerService(); const source = useGetSourceFromSearchParams(); + const destination = useGetDestinationFromSearchParams(); // if the user is navigating from the first step the form may be dirty useMount(() => { @@ -88,7 +92,8 @@ const SimplifiedConnectionCreationConfigureConnection: React.FC = () => { return ( ); @@ -96,8 +101,8 @@ const SimplifiedConnectionCreationConfigureConnection: React.FC = () => { const FirstNav: React.FC = () => { const createLink = useCurrentWorkspaceLink(); - const source = useGetSourceFromSearchParams(); const destination = useGetDestinationFromSearchParams(); + const source = useGetSourceFromSearchParams(); const { isValid, errors } = useFormState(); const { trigger } = useFormContext(); diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx index 98519b6fbfe..08d41fbb492 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx @@ -8,6 +8,7 @@ import { FlexContainer } from "components/ui/Flex"; import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; +import { DestinationRead, SourceRead } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import { useExperiment } from "hooks/services/Experiment"; @@ -24,14 +25,16 @@ import { SimplifiedSchemaChangeNotificationFormField } from "./SimplifiedSchemaC interface SimplifiedConnectionsSettingsCardProps { title: string; isCreating: boolean; - sourceName: string; + source: SourceRead; + destination: DestinationRead; isDeprecated?: boolean; } export const SimplifiedConnectionsSettingsCard: React.FC = ({ title, isCreating, - sourceName, + source, + destination, isDeprecated = false, }) => { const [isAdvancedOpen, setIsAdvancedOpen] = useState(false); @@ -43,7 +46,9 @@ export const SimplifiedConnectionsSettingsCard: React.FC - {isCreating && } + {isCreating && ( + + )} {isCreating && } @@ -79,7 +84,8 @@ export const SimplifiedConnectionsSettingsCard: React.FC )} diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.module.scss b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.module.scss new file mode 100644 index 00000000000..a9a255ac590 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.module.scss @@ -0,0 +1,14 @@ +@use "scss/variables"; + +.originalCasing, +%originalCasing { + text-transform: unset; +} + +.sourceNamespace { + @extend %originalCasing; + + &:not(:first-child) { + margin-left: variables.$spacing-xs; + } +} diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx index 499d5abb33b..fdd5f6e7582 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx @@ -1,3 +1,4 @@ +import get from "lodash/get"; import { ComponentProps, useEffect } from "react"; import { Controller, useFormContext, useFormState, useWatch } from "react-hook-form"; import { FormattedMessage, useIntl } from "react-intl"; @@ -5,53 +6,134 @@ import { FormattedMessage, useIntl } from "react-intl"; import { FormConnectionFormValues } from "components/connection/ConnectionForm/formConfig"; import { FormFieldLayout } from "components/connection/ConnectionForm/FormFieldLayout"; import { RadioButtonTiles } from "components/connection/CreateConnection/RadioButtonTiles"; -import { FormControl } from "components/forms"; import { ControlLabels } from "components/LabeledControl"; +import { Badge } from "components/ui/Badge"; import { Box } from "components/ui/Box"; import { FlexContainer } from "components/ui/Flex"; +import { Input } from "components/ui/Input"; import { ExternalLink } from "components/ui/Link"; import { ListBox } from "components/ui/ListBox"; import { Text } from "components/ui/Text"; -import { NamespaceDefinitionType } from "core/api/types/AirbyteClient"; +import { DestinationRead, NamespaceDefinitionType, SourceRead } from "core/api/types/AirbyteClient"; import { links } from "core/utils/links"; +import { naturalComparator } from "core/utils/objects"; +import { SourceNamespaceConfiguration, DestinationNamespaceConfiguration } from "./ConnectorNamespaceConfiguration"; import { InputContainer } from "./InputContainer"; +import styles from "./SimplifiedDestinationNamespaceFormField.module.scss"; + +// eslint-disable-next-line no-template-curly-in-string +const SOURCE_NAMESPACE_REPLACEMENT_STRING = "${SOURCE_NAMESPACE}"; export const SimplifiedDestinationNamespaceFormField: React.FC<{ isCreating: boolean; - sourceName: string; + source: SourceRead; + destination: DestinationRead; disabled?: boolean; -}> = ({ isCreating, sourceName, disabled }) => { +}> = ({ isCreating, source, destination, disabled }) => { const { trigger, setValue, control, watch } = useFormContext(); const { defaultValues } = useFormState(); const namespaceDefinition = useWatch({ name: "namespaceDefinition", control }); + const streams = useWatch({ name: "syncCatalog.streams", control }); + const namespaceFormat = useWatch({ name: "namespaceFormat", control }); const { formatMessage } = useIntl(); const watchedNamespaceDefinition = watch("namespaceDefinition"); useEffect(() => { - if (watchedNamespaceDefinition === NamespaceDefinitionType.customformat) { - setValue("namespaceFormat", defaultValues?.namespaceFormat, { shouldDirty: true }); - } - trigger("namespaceFormat", { shouldFocus: true }); }, [trigger, setValue, defaultValues?.namespaceFormat, watchedNamespaceDefinition]); - const watchedNamespaceFormat = watch("namespaceFormat"); - useEffect(() => { - trigger("namespaceFormat"); - }, [trigger, watchedNamespaceFormat]); + const sourceNamespaceAbilities = SourceNamespaceConfiguration[source.sourceDefinitionId] ?? { + supportsNamespaces: true, + }; + const destinationNamespaceAbilities = DestinationNamespaceConfiguration[destination.destinationDefinitionId] ?? { + supportsNamespaces: true, + }; + + if (!destinationNamespaceAbilities.supportsNamespaces) { + return null; + } + + const destinationDefinedNamespace = + (destinationNamespaceAbilities.defaultNamespacePath && + get(destination.connectionConfiguration, destinationNamespaceAbilities.defaultNamespacePath)) ?? + "no_value_provided"; + + const destinationDefinedDescriptionValues = { + destinationDefinedNamespace, + badge: (children: React.ReactNode[]) => ( + + {children} + + ), + }; + + const enabledStreamNamespaces = Array.from( + streams.reduce((acc, stream) => { + if (stream.config?.selected && stream.stream?.namespace) { + acc.add(stream.stream.namespace); + } + return acc; + }, new Set()) + ).sort(naturalComparator); + + const sourceDefinedDescriptionValues = { + sourceDefinedNamespaces: enabledStreamNamespaces, + badges: (children: React.ReactNode[]) => { + if (children.length === 0) { + return null; + } + + return ( + + {children.map((child) => ( + + {child} + + ))} + + ); + }, + }; const customFormatField = namespaceDefinition === NamespaceDefinitionType.customformat ? ( - + <> + ( + <> + + + + {fieldState.error && ( + + + + + + )} + + )} + /> + {namespaceFormat?.includes(SOURCE_NAMESPACE_REPLACEMENT_STRING) && + enabledStreamNamespaces.map((namespace) => { + return ( + + {namespaceFormat.replace(SOURCE_NAMESPACE_REPLACEMENT_STRING, namespace)} + + ); + })} + ) : null; const destinationNamespaceOptions: ComponentProps>["options"] = [ @@ -67,12 +149,20 @@ export const SimplifiedDestinationNamespaceFormField: React.FC<{ ? "connectionForm.destinationFormatNext" : formatMessage({ id: "connectionForm.destinationFormatNext" }), description: "connectionForm.destinationFormatDescriptionNext", + descriptionValues: destinationDefinedDescriptionValues, }, - { - value: NamespaceDefinitionType.source, - label: isCreating ? "connectionForm.sourceFormatNext" : formatMessage({ id: "connectionForm.sourceFormatNext" }), - description: "connectionForm.sourceFormatDescriptionNext", - }, + ...(sourceNamespaceAbilities.supportsNamespaces + ? [ + { + value: NamespaceDefinitionType.source, + label: isCreating + ? "connectionForm.sourceFormatNext" + : formatMessage({ id: "connectionForm.sourceFormatNext" }), + description: "connectionForm.sourceFormatDescriptionNext", + descriptionValues: sourceDefinedDescriptionValues, + }, + ] + : []), ]; return ( @@ -119,14 +209,20 @@ export const SimplifiedDestinationNamespaceFormField: React.FC<{ {field.value === NamespaceDefinitionType.destination && ( - + )} {field.value === NamespaceDefinitionType.source && ( - + )} diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationStreamPrefixNameFormField.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationStreamPrefixNameFormField.tsx index 49e8b430166..7034938fcb3 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationStreamPrefixNameFormField.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationStreamPrefixNameFormField.tsx @@ -30,6 +30,10 @@ export const SimplifiedDestinationStreamPrefixNameFormField: React.FC<{ disabled +   + + + diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedSchemaQuestionnaire.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedSchemaQuestionnaire.tsx index 6eb30ca66df..db8fd42153a 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedSchemaQuestionnaire.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedSchemaQuestionnaire.tsx @@ -14,6 +14,7 @@ import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; import { DestinationSyncMode, SyncMode } from "core/api/types/AirbyteClient"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import styles from "./SimplifiedSchemaQuestionnaire.module.scss"; @@ -98,6 +99,7 @@ export const getEnforcedIncrementOrRefresh = (supportedSyncModes: SyncMode[]) => }; export const SimplifiedSchemaQuestionnaire = () => { + const analyticsService = useAnalyticsService(); const { connection, destDefinitionSpecification: { supportedDestinationSyncModes }, @@ -139,19 +141,40 @@ export const SimplifiedSchemaQuestionnaire = () => { const enforcedIncrementOrRefresh = getEnforcedIncrementOrRefresh(supportedSyncModes); const [selectedDelivery, _setSelectedDelivery] = useState(enforcedSelectedDelivery); - const [selectedIncrementOrRefresh, setSelectedIncrementOrRefresh] = useState( + const [selectedIncrementOrRefresh, _setSelectedIncrementOrRefresh] = useState( enforcedIncrementOrRefresh ); const setSelectedDelivery: typeof _setSelectedDelivery = (value) => { + analyticsService.track(Namespace.SYNC_QUESTIONNAIRE, Action.ANSWERED, { + actionDescription: "First question has been answered", + question: "delivery", + answer: value, + }); + _setSelectedDelivery(value); if (value === "mirrorSource") { // clear any user-provided answer for the second question when switching to mirrorSource // this is purely a UX decision - setSelectedIncrementOrRefresh(enforcedIncrementOrRefresh); + setSelectedIncrementOrRefresh(enforcedIncrementOrRefresh, { automatedAction: true }); } }; + const setSelectedIncrementOrRefresh = ( + value: SyncMode | undefined, + { automatedAction }: { automatedAction?: boolean } = { automatedAction: false } + ) => { + if (!automatedAction) { + analyticsService.track(Namespace.SYNC_QUESTIONNAIRE, Action.ANSWERED, { + actionDescription: "Second question has been answered", + question: "all_or_some", + answer: value, + }); + } + + _setSelectedIncrementOrRefresh(value); + }; + const selectedModes = useMemo(() => { if (selectedDelivery === "mirrorSource") { return questionnaireOutcomes.mirrorSource.map(([syncMode, destinationSyncMode]) => { @@ -208,16 +231,41 @@ export const SimplifiedSchemaQuestionnaire = () => { }); setValue("syncCatalog.streams", nextFields); trigger("syncCatalog.streams"); - }, [setValue, trigger, getValues, selectedDelivery, selectedIncrementOrRefresh, selectedModes]); + analyticsService.track(Namespace.SYNC_QUESTIONNAIRE, Action.APPLIED, { + actionDescription: "Questionnaire has applied a sync mode", + delivery: selectedDelivery, + all_or_some: selectedIncrementOrRefresh, + }); + }, [setValue, trigger, getValues, selectedDelivery, selectedIncrementOrRefresh, selectedModes, analyticsService]); + + const showFirstQuestion = enforcedSelectedDelivery == null; const showSecondQuestion = enforcedIncrementOrRefresh == null && selectedDelivery === "appendChanges"; + useEffect(() => { + if (showFirstQuestion) { + analyticsService.track(Namespace.SYNC_QUESTIONNAIRE, Action.DISPLAYED, { + actionDescription: "First question has been shown to the user", + question: "delivery", + }); + } + }, [showFirstQuestion, analyticsService]); + + useEffect(() => { + if (showSecondQuestion) { + analyticsService.track(Namespace.SYNC_QUESTIONNAIRE, Action.DISPLAYED, { + actionDescription: "Second question has been shown to the user", + question: "all_or_some", + }); + } + }, [showSecondQuestion, analyticsService]); + return ( - {enforcedSelectedDelivery == null && ( + {showFirstQuestion && ( { - const { formatMessage } = useIntl(); - const { - connection, - destDefinitionSpecification: { supportedDestinationSyncModes }, - } = useConnectionFormService(); - - const streamSupportedSyncModes: SyncMode[] = useMemo(() => { - const foundModes = new Set(); - for (let i = 0; i < connection.syncCatalog.streams.length; i++) { - const stream = connection.syncCatalog.streams[i]; - stream.stream?.supportedSyncModes?.forEach((mode) => foundModes.add(mode)); - } - return Array.from(foundModes); - }, [connection.syncCatalog.streams]); - - const availableSyncModes: SyncModeValue[] = useMemo( - () => - SUPPORTED_MODES.filter( - ([syncMode, destinationSyncMode]) => - streamSupportedSyncModes.includes(syncMode) && supportedDestinationSyncModes?.includes(destinationSyncMode) - ).map(([syncMode, destinationSyncMode]) => ({ - syncMode, - destinationSyncMode, - })), - [streamSupportedSyncModes, supportedDestinationSyncModes] - ); - - const syncModeOptions: Array> = useMemo( - () => - availableSyncModes.map((option) => { - const syncModeId = option.syncMode === SyncMode.full_refresh ? "syncMode.fullRefresh" : "syncMode.incremental"; - const destinationSyncModeId = - option.destinationSyncMode === DestinationSyncMode.overwrite - ? "destinationSyncMode.overwrite" - : option.destinationSyncMode === DestinationSyncMode.append_dedup - ? "destinationSyncMode.appendDedup" - : "destinationSyncMode.append"; - return { - label: `${formatMessage({ id: syncModeId })} | ${formatMessage({ - id: destinationSyncModeId, - })}`, - value: option, - }; - }), - [formatMessage, availableSyncModes] - ); - - const [defaultSyncMode, setDefaultSyncMode] = useState(availableSyncModes[0]); - - return ( - - - {formatMessage({ id: "form.syncMode" })} - - {formatMessage({ id: "form.syncMode.subtitle" })} - - - } - /> - - - ); -}; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap index 9b87244e172..a1e2bcb3ef4 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap @@ -1025,69 +1025,13 @@ exports[`CreateConnectionForm should render 1`] = `
-
- -
-
-
+ />
-
-
-

- All -

-
-
-
+ />
diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/useAnalyticsTrackFunctions.ts b/airbyte-webapp/src/components/connection/CreateConnectionForm/useAnalyticsTrackFunctions.ts index 767e54ddfd5..9dcfd9ee0e4 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/useAnalyticsTrackFunctions.ts +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/useAnalyticsTrackFunctions.ts @@ -1,6 +1,6 @@ import { useCallback } from "react"; -import { SchemaError } from "core/api"; +import { ErrorWithJobInfo } from "core/api"; import { DestinationRead, SourceRead } from "core/api/types/AirbyteClient"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; @@ -11,17 +11,19 @@ export const useAnalyticsTrackFunctions = () => { const analyticsService = useAnalyticsService(); const trackFailure = useCallback( - (source: SourceRead, destination: DestinationRead, schemaError: SchemaError) => + (source: SourceRead, destination: DestinationRead, schemaError: Error | ErrorWithJobInfo) => { + const jobInfo = ErrorWithJobInfo.getJobInfo(schemaError); analyticsService.track(Namespace.CONNECTION, Action.DISCOVER_SCHEMA, { actionDescription: "Discover schema failure", connector_source_definition: source.sourceName, connector_source_definition_id: source.sourceDefinitionId, connector_destination_definition: destination.destinationName, connector_destination_definition_id: destination.destinationDefinitionId, - failure_type: schemaError?.response?.failureReason?.failureType, - failure_external_message: schemaError?.response?.failureReason?.externalMessage, - failure_internal_message: schemaError?.response?.failureReason?.internalMessage, - }), + failure_type: jobInfo?.failureReason?.failureType, + failure_external_message: jobInfo?.failureReason?.externalMessage, + failure_internal_message: jobInfo?.failureReason?.internalMessage, + }); + }, [analyticsService] ); diff --git a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx index 66395a20821..a80d67c7221 100644 --- a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx +++ b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx @@ -52,21 +52,17 @@ const destinationNamespaceValidationSchema = yup.object().shape({ interface DestinationNamespaceModalProps { initialValues: Pick; - onCloseModal: () => void; - onSubmit: (values: DestinationNamespaceFormValues) => void; + onCancel: () => void; + onSubmit: (values: DestinationNamespaceFormValues) => Promise; } export const DestinationNamespaceModal: React.FC = ({ initialValues, - onCloseModal, + onCancel, onSubmit, }) => { const { formatMessage } = useIntl(); - const onSubmitCallback = async (values: DestinationNamespaceFormValues) => { - onCloseModal(); - onSubmit(values); - }; return (
namespaceFormat: initialValues.namespaceFormat ?? "${SOURCE_NAMESPACE}", }} schema={destinationNamespaceValidationSchema} - onSubmit={onSubmitCallback} + onSubmit={onSubmit} > <> @@ -112,7 +108,7 @@ export const DestinationNamespaceModal: React.FC diff --git a/airbyte-webapp/src/components/connection/DestinationStreamNamesModal/DestinationStreamNamesModal.tsx b/airbyte-webapp/src/components/connection/DestinationStreamNamesModal/DestinationStreamNamesModal.tsx index f0114699420..b9dcc551a47 100644 --- a/airbyte-webapp/src/components/connection/DestinationStreamNamesModal/DestinationStreamNamesModal.tsx +++ b/airbyte-webapp/src/components/connection/DestinationStreamNamesModal/DestinationStreamNamesModal.tsx @@ -70,22 +70,17 @@ const destinationStreamNamesValidationSchema = yup.object().shape({ interface DestinationStreamNamesModalProps { initialValues: Pick; - onCloseModal: () => void; - onSubmit: (value: DestinationStreamNamesFormValues) => void; + onCancel: () => void; + onSubmit: (value: DestinationStreamNamesFormValues) => Promise; } export const DestinationStreamNamesModal: React.FC = ({ initialValues, - onCloseModal, + onCancel, onSubmit, }) => { const { formatMessage } = useIntl(); - const onSubmitCallback = async (values: DestinationStreamNamesFormValues) => { - onCloseModal(); - onSubmit(values); - }; - return ( @@ -135,7 +130,7 @@ export const DestinationStreamNamesModal: React.FC diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.module.scss b/airbyte-webapp/src/components/connection/JobProgress/JobProgress.module.scss deleted file mode 100644 index 21e0315ae22..00000000000 --- a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.module.scss +++ /dev/null @@ -1,23 +0,0 @@ -@use "scss/variables"; -@use "scss/colors"; - -.estimationStats { - display: flex; - justify-content: space-between; -} - -.estimationDetails { - display: flex; - gap: variables.$spacing-lg; - margin: variables.$spacing-md 0; - - .icon { - color: colors.$grey-400; - margin-right: variables.$spacing-sm; - } -} - -.streams { - margin: variables.$spacing-md 0 0; - width: 100%; -} diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx b/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx deleted file mode 100644 index ed2fbaf8ee6..00000000000 --- a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx +++ /dev/null @@ -1,125 +0,0 @@ -import classNames from "classnames"; -import { FormattedMessage, useIntl } from "react-intl"; - -import { Icon } from "components/ui/Icon"; -import { Text } from "components/ui/Text"; - -import { JobWithAttempts } from "area/connection/types/jobs"; -import { getJobStatus } from "area/connection/utils/jobs"; -import { AttemptRead, AttemptStatus, SynchronousJobRead } from "core/api/types/AirbyteClient"; -import { formatBytes } from "core/utils/numberHelper"; - -import styles from "./JobProgress.module.scss"; -import { ProgressLine } from "./JobProgressLine"; -import { StreamProgress } from "./StreamProgress"; -import { progressBarCalculations } from "./utils"; - -function isJobsWithJobs(job: JobWithAttempts | SynchronousJobRead): job is JobWithAttempts { - return "attempts" in job; -} - -interface ProgressBarProps { - job: JobWithAttempts | SynchronousJobRead; - expanded?: boolean; -} - -export const JobProgress: React.FC = ({ job, expanded }) => { - const { formatMessage, formatNumber } = useIntl(); - - let latestAttempt: AttemptRead | undefined; - if (isJobsWithJobs(job) && job.attempts) { - latestAttempt = job.attempts[job.attempts.length - 1]; - } - if (!latestAttempt) { - return null; - } - - const jobStatus = getJobStatus(job); - if (["failed", "succeeded", "cancelled"].includes(jobStatus)) { - return null; - } - - const { - displayProgressBar, - totalPercentRecords, - timeRemaining, - numeratorBytes, - numeratorRecords, - denominatorRecords, - denominatorBytes, - elapsedTimeMS, - } = progressBarCalculations(latestAttempt); - - let timeRemainingString = ""; - if (elapsedTimeMS && timeRemaining) { - const minutesRemaining = Math.ceil(timeRemaining / 1000 / 60); - const hoursRemaining = Math.ceil(minutesRemaining / 60); - if (minutesRemaining <= 60) { - timeRemainingString = formatMessage({ id: "connection.progress.minutesRemaining" }, { value: minutesRemaining }); - } else { - timeRemainingString = formatMessage({ id: "connection.progress.hoursRemaining" }, { value: hoursRemaining }); - } - } - - return ( - - {displayProgressBar && ( - - )} - {latestAttempt?.status === AttemptStatus.running && ( - <> - {displayProgressBar && ( -
- {timeRemaining < Infinity && timeRemaining > 0 && timeRemainingString} - {formatNumber(totalPercentRecords, { style: "percent", maximumFractionDigits: 0 })} -
- )} - {expanded && ( - <> - {denominatorRecords > 0 && denominatorBytes > 0 && ( -
- - - - - - - - -
- )} - {latestAttempt.streamStats && ( -
- {latestAttempt.streamStats - ?.map((stats) => ({ - ...stats, - done: (stats.stats.recordsEmitted ?? 0) >= (stats.stats.estimatedRecords ?? Infinity), - })) - // Move finished streams to the end of the list - .sort((a, b) => Number(a.done) - Number(b.done)) - .map((stream) => { - return ; - })} -
- )} - - )} - - )} - - ); -}; diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.module.scss b/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.module.scss deleted file mode 100644 index d23bb032cb3..00000000000 --- a/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.module.scss +++ /dev/null @@ -1,24 +0,0 @@ -@use "scss/colors"; - -.lineOuter { - height: 5px; - width: 100%; - background-color: colors.$grey-100; - border-radius: 10px; - margin-top: 5px; - margin-bottom: 5px; -} - -.lineInner { - height: 100%; - border-radius: 10px; - transition: width 5s ease-in-out; - - &.warning { - background-color: colors.$yellow-400; - } - - &.default { - background-color: colors.$blue-200; - } -} diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.tsx b/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.tsx deleted file mode 100644 index 028b730261e..00000000000 --- a/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.tsx +++ /dev/null @@ -1,29 +0,0 @@ -// Inspired by https://dev.to/ramonak/react-how-to-create-a-custom-progress-bar-component-in-5-minutes-2lcl - -import classNames from "classnames"; -import { useIntl } from "react-intl"; - -import styles from "./JobProgressLine.module.scss"; - -interface ProgressLineProps { - type?: "default" | "warning"; - percent: number; -} - -export const ProgressLine: React.FC = ({ type = "default", percent }) => { - const { formatMessage } = useIntl(); - return ( -
-
-
- ); -}; diff --git a/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.module.scss b/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.module.scss deleted file mode 100644 index 04f4f053246..00000000000 --- a/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.module.scss +++ /dev/null @@ -1,73 +0,0 @@ -@use "scss/variables"; -@use "scss/colors"; - -.stream { - display: inline-block; - padding: variables.$spacing-xs variables.$spacing-md; - background: colors.$blue-50; - border-radius: 16px; - margin-right: variables.$spacing-md; - margin-bottom: variables.$spacing-sm; - white-space: nowrap; - color: colors.$grey-700; - line-height: 16px; -} - -.wrapper { - display: flex; - align-items: center; - gap: variables.$spacing-sm; - min-height: 16px + 2 * variables.$spacing-xs; -} - -.progress { - justify-content: center; - align-items: center; - min-width: 16px; - margin: variables.$spacing-xs; - aspect-ratio: 1 / 1; - display: flex; - - .check { - fill: colors.$foreground; - display: none; - } - - .fg { - stroke: colors.$blue; - } - - .bg { - fill: colors.$foreground; - } - - &.done { - .bg { - fill: colors.$green; - } - - .fg { - display: none; - } - - .check { - display: block; - } - } -} - -.metrics { - margin: variables.$spacing-md 0 0; - display: grid; - grid-template-columns: max-content auto; - gap: variables.$spacing-md; - - dt { - grid-column-start: 1; - } - - dd { - margin: 0; - grid-column-start: 2; - } -} diff --git a/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.tsx b/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.tsx deleted file mode 100644 index 4a9bd328b9c..00000000000 --- a/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.tsx +++ /dev/null @@ -1,96 +0,0 @@ -import classNames from "classnames"; -import { FormattedMessage, FormattedNumber, useIntl } from "react-intl"; - -import { Text } from "components/ui/Text"; -import { Tooltip } from "components/ui/Tooltip"; - -import { AttemptStreamStats } from "core/api/types/AirbyteClient"; - -import styles from "./StreamProgress.module.scss"; - -interface StreamProgressProps { - stream: AttemptStreamStats; -} - -const CircleProgress: React.FC<{ percent: number }> = ({ percent }) => { - const svgClassName = classNames(styles.progress, { - [styles.done]: percent >= 1, - }); - - return ( - - - - - - ); -}; - -export const StreamProgress: React.FC = ({ stream }) => { - const { formatNumber } = useIntl(); - const { recordsEmitted, estimatedRecords } = stream.stats; - - const progress = estimatedRecords ? (recordsEmitted ?? 0) / estimatedRecords : undefined; - - return ( - - - {stream.streamName} {progress !== undefined && } - - - } - > - - - -
-
- -
-
- {progress !== undefined ? ( - - ) : ( - - )} -
- {(estimatedRecords || recordsEmitted) && ( - <> -
- -
-
- {estimatedRecords ? ( - - ) : ( - - )} -
- - )} -
- - ); -}; diff --git a/airbyte-webapp/src/components/connection/JobProgress/index.tsx b/airbyte-webapp/src/components/connection/JobProgress/index.tsx deleted file mode 100644 index 5e1783a4557..00000000000 --- a/airbyte-webapp/src/components/connection/JobProgress/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export * from "./JobProgress"; diff --git a/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts b/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts deleted file mode 100644 index 6339f209040..00000000000 --- a/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { AttemptRead, AttemptStats, AttemptStatus, AttemptStreamStats } from "core/api/types/AirbyteClient"; - -import { progressBarCalculations } from "./utils"; - -describe("#progressBarCalculations", () => { - beforeEach(() => { - jest.spyOn(Date, "now").mockImplementation(() => new Date("2023-01-01T00:00:00.000Z").getTime()); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it("for an attempt with no throughput information", () => { - const attempt = makeAttempt(); - const { displayProgressBar, totalPercentRecords } = progressBarCalculations(attempt); - - expect(displayProgressBar).toEqual(false); - expect(totalPercentRecords).toEqual(0); - }); - - it("for an attempt with total stats", () => { - const totalStats: AttemptStats = { recordsEmitted: 1, estimatedRecords: 100, bytesEmitted: 1, estimatedBytes: 50 }; - const attempt = makeAttempt(totalStats); - const { displayProgressBar, totalPercentRecords, elapsedTimeMS, timeRemaining } = progressBarCalculations(attempt); - - expect(displayProgressBar).toEqual(true); - expect(totalPercentRecords).toEqual(0.01); - expect(elapsedTimeMS).toEqual(10 * 1000); - expect(timeRemaining).toEqual(990 * 1000); - }); - - it("for an attempt with per-stream stats", () => { - const totalStats: AttemptStats = { recordsEmitted: 3, estimatedRecords: 300, bytesEmitted: 3, estimatedBytes: 300 }; - const streamStatsA: AttemptStreamStats = { - streamName: "A", - stats: { recordsEmitted: 1, estimatedRecords: 100, bytesEmitted: 1, estimatedBytes: 100 }, - }; - const streamStatsB: AttemptStreamStats = { - streamName: "B", - stats: { recordsEmitted: 2, estimatedRecords: 100, bytesEmitted: 2, estimatedBytes: 100 }, - }; - const streamStatsC: AttemptStreamStats = { - streamName: "C", - stats: {}, - }; - - const attempt = makeAttempt(totalStats, [streamStatsA, streamStatsB, streamStatsC]); - const { displayProgressBar, totalPercentRecords, elapsedTimeMS, timeRemaining } = progressBarCalculations(attempt); - - expect(displayProgressBar).toEqual(true); - expect(totalPercentRecords).toEqual(0.01); - expect(elapsedTimeMS).toEqual(10 * 1000); - expect(timeRemaining).toEqual(990 * 1000); - }); -}); - -const makeAttempt = (totalStats: AttemptStats = {}, streamStats: AttemptStreamStats[] = []) => { - const now = Date.now(); - // API returns time in seconds - const createdAt = now / 1000 - 10; - const updatedAt = now / 1000; - const id = 123; - const status: AttemptStatus = "running"; - const attempt: AttemptRead = { id, status, createdAt, updatedAt, totalStats, streamStats }; - return attempt; -}; diff --git a/airbyte-webapp/src/components/connection/JobProgress/utils.ts b/airbyte-webapp/src/components/connection/JobProgress/utils.ts deleted file mode 100644 index ce6435307d2..00000000000 --- a/airbyte-webapp/src/components/connection/JobProgress/utils.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { AttemptRead, AttemptStatus } from "core/api/types/AirbyteClient"; - -export const progressBarCalculations = (latestAttempt: AttemptRead) => { - let numeratorRecords = 0; - let denominatorRecords = 0; - let totalPercentRecords = 0; - let numeratorBytes = 0; - let denominatorBytes = 0; - let elapsedTimeMS = 0; - let timeRemaining = 0; - let displayProgressBar = true; - - if ( - latestAttempt.totalStats?.recordsEmitted !== undefined && - latestAttempt.totalStats?.estimatedRecords !== undefined && - latestAttempt.totalStats?.bytesEmitted !== undefined && - latestAttempt.totalStats?.estimatedBytes !== undefined - ) { - numeratorRecords = latestAttempt.totalStats.recordsEmitted; - denominatorRecords = latestAttempt.totalStats.estimatedRecords; - numeratorBytes = latestAttempt.totalStats.bytesEmitted; - denominatorBytes = latestAttempt.totalStats.estimatedBytes; - } else if (latestAttempt.streamStats) { - for (const stream of latestAttempt.streamStats) { - numeratorRecords += stream.stats.recordsEmitted ?? 0; - denominatorRecords += stream.stats.estimatedRecords ?? 0; - numeratorBytes += stream.stats.bytesEmitted ?? 0; - denominatorBytes += stream.stats.estimatedBytes ?? 0; - } - } - - totalPercentRecords = denominatorRecords > 0 ? numeratorRecords / denominatorRecords : 0; - - // chose to estimate time remaining based on records rather than bytes - if (latestAttempt.status === AttemptStatus.running && denominatorRecords > 0) { - elapsedTimeMS = Date.now() - latestAttempt.createdAt * 1000; - timeRemaining = Math.floor(elapsedTimeMS / totalPercentRecords) * (1 - totalPercentRecords); // in ms - } else { - displayProgressBar = false; - } - - return { - displayProgressBar, - totalPercentRecords, - timeRemaining, - numeratorBytes, - numeratorRecords, - denominatorRecords, - denominatorBytes, - elapsedTimeMS, - }; -}; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx index 036fd8eace3..dc26f956731 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx @@ -1,4 +1,4 @@ -import React, { ReactNode, useMemo } from "react"; +import React, { ReactNode } from "react"; import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; @@ -8,17 +8,16 @@ import { Switch } from "components/ui/Switch"; import { Text } from "components/ui/Text"; import { AirbyteStream, AirbyteStreamConfiguration } from "core/api/types/AirbyteClient"; -import { useExperiment } from "hooks/services/Experiment"; import styles from "./StreamPanelHeader.module.scss"; import { SyncModeSelect, SyncModeValue } from "../SyncModeSelect"; interface StreamPanelHeaderProps { - config?: AirbyteStreamConfiguration; + stream: AirbyteStream; + config: AirbyteStreamConfiguration; disabled?: boolean; onClose: () => void; onSelectedChange: () => void; - stream?: AirbyteStream; onSelectSyncMode: (option: SyncModeValue) => void; availableSyncModes: SyncModeValue[]; } @@ -41,21 +40,9 @@ export const StreamProperty: React.FC = ({ messageId, value ); const NamespaceProperty: React.FC<{ namespace?: string }> = ({ namespace }) => { - const isSimplifiedCatalogRowEnabled = useExperiment("connection.syncCatalog.simplifiedCatalogRow", true); - - if (isSimplifiedCatalogRowEnabled) { - return namespace ? ( - - ) : null; - } - - return ( - } - data-testid="stream-details-namespace" - /> - ); + return namespace ? ( + + ) : null; }; export const StreamPanelHeader: React.FC = ({ @@ -67,37 +54,19 @@ export const StreamPanelHeader: React.FC = ({ availableSyncModes, onSelectSyncMode, }) => { - const isSimplifiedCatalogRowEnabled = useExperiment("connection.syncCatalog.simplifiedCatalogRow", true); - - const syncSchema: SyncModeValue | undefined = useMemo(() => { - if (!config) { - return undefined; - } - const { syncMode, destinationSyncMode } = config; - return { syncMode, destinationSyncMode }; - }, [config]); - - const syncMode = ( - <> - {config?.syncMode && } - {` | `} - {config?.destinationSyncMode && } - - ); + const { syncMode, destinationSyncMode, selected: isStreamSelectedForSync } = config ?? {}; return ( -
- -
+ @@ -105,19 +74,17 @@ export const StreamPanelHeader: React.FC = ({ - - {isSimplifiedCatalogRowEnabled ? ( + {isStreamSelectedForSync && ( + - ) : ( - - )} - + + )} - + )} + diff --git a/airbyte-webapp/src/components/settings/SettingsNavigation/index.ts b/airbyte-webapp/src/components/settings/SettingsNavigation/index.ts deleted file mode 100644 index 87c0546437f..00000000000 --- a/airbyte-webapp/src/components/settings/SettingsNavigation/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./SettingsNavigation"; diff --git a/airbyte-webapp/src/views/Connector/RequestConnectorModal/RequestConnectorModal.tsx b/airbyte-webapp/src/components/source/SelectConnector/RequestConnectorModal.tsx similarity index 90% rename from airbyte-webapp/src/views/Connector/RequestConnectorModal/RequestConnectorModal.tsx rename to airbyte-webapp/src/components/source/SelectConnector/RequestConnectorModal.tsx index 7c5f532e995..21e1c22b79a 100644 --- a/airbyte-webapp/src/views/Connector/RequestConnectorModal/RequestConnectorModal.tsx +++ b/airbyte-webapp/src/components/source/SelectConnector/RequestConnectorModal.tsx @@ -10,8 +10,6 @@ import { ModalBody, ModalFooter } from "components/ui/Modal"; import { useNotificationService } from "hooks/services/Notification"; import useRequestConnector from "hooks/services/useRequestConnector"; -import { Values } from "./types"; - interface ConnectorRequest { connectorType: "source" | "destination"; name: string; @@ -20,7 +18,8 @@ interface ConnectorRequest { } interface RequestConnectorModalProps { - onClose: () => void; + onSubmit: () => void; + onCancel: () => void; connectorType: ConnectorRequest["connectorType"]; workspaceEmail?: string; searchedConnectorName?: string; @@ -35,8 +34,9 @@ const validationSchema = yup.object().shape({ const RequestControl = FormControl; -const RequestConnectorModal: React.FC = ({ - onClose, +export const RequestConnectorModal: React.FC = ({ + onSubmit, + onCancel, connectorType, searchedConnectorName, workspaceEmail, @@ -45,14 +45,14 @@ const RequestConnectorModal: React.FC = ({ const notificationService = useNotificationService(); const { requestConnector } = useRequestConnector(); - const onSubmit = (values: Values) => { + const onSubmitBtnClick = async (values: ConnectorRequest) => { requestConnector(values); notificationService.registerNotification({ id: "connector.requestConnector.success", text: formatMessage({ id: "connector.request.success" }), type: "success", }); - onClose(); + onSubmit(); }; return ( @@ -64,9 +64,7 @@ const RequestConnectorModal: React.FC = ({ email: workspaceEmail ?? "", }} schema={validationSchema} - onSubmit={async (values) => { - onSubmit(values); - }} + onSubmit={onSubmitBtnClick} trackDirtyChanges > @@ -96,7 +94,7 @@ const RequestConnectorModal: React.FC = ({ @@ -119,5 +117,3 @@ const NameControl = () => { /> ); }; - -export default RequestConnectorModal; diff --git a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx index 92cbde0b343..ab49d1e1a9c 100644 --- a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx +++ b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx @@ -16,10 +16,10 @@ import { isSourceDefinition } from "core/domain/connector/source"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useLocalStorage } from "core/utils/useLocalStorage"; import { useModalService } from "hooks/services/Modal"; -import RequestConnectorModal from "views/Connector/RequestConnectorModal"; import { ConnectorGrid } from "./ConnectorGrid"; import { FilterSupportLevel } from "./FilterSupportLevel"; +import { RequestConnectorModal } from "./RequestConnectorModal"; import styles from "./SelectConnector.module.scss"; import { useTrackSelectConnector } from "./useTrackSelectConnector"; @@ -46,7 +46,7 @@ const SelectConnectorSupportLevel: React.FC = ({ }) => { const { formatMessage } = useIntl(); const { email } = useCurrentWorkspace(); - const { openModal, closeModal } = useModalService(); + const { openModal } = useModalService(); const trackSelectConnector = useTrackSelectConnector(connectorType); const [searchTerm, setSearchTerm] = useState(""); const [supportLevelsInLocalStorage, setSelectedSupportLevels] = useLocalStorage( @@ -81,14 +81,15 @@ const SelectConnectorSupportLevel: React.FC = ({ }; const onOpenRequestConnectorModal = () => - openModal({ + openModal({ title: formatMessage({ id: "connector.requestConnector" }), - content: () => ( + content: ({ onComplete, onCancel }) => ( ), size: "sm", diff --git a/airbyte-webapp/src/components/ui/Banner/AlertBanner.module.scss b/airbyte-webapp/src/components/ui/Banner/AlertBanner.module.scss index 7cc28db18ed..ed8d2ebe1a0 100644 --- a/airbyte-webapp/src/components/ui/Banner/AlertBanner.module.scss +++ b/airbyte-webapp/src/components/ui/Banner/AlertBanner.module.scss @@ -20,7 +20,7 @@ color: colors.$white; } -.red { - background-color: colors.$red-400; - color: colors.$white; +.yellow { + background-color: colors.$yellow-500; + color: colors.$black; } diff --git a/airbyte-webapp/src/components/ui/Banner/AlertBanner.tsx b/airbyte-webapp/src/components/ui/Banner/AlertBanner.tsx index fe815af509e..d59afff4568 100644 --- a/airbyte-webapp/src/components/ui/Banner/AlertBanner.tsx +++ b/airbyte-webapp/src/components/ui/Banner/AlertBanner.tsx @@ -11,7 +11,7 @@ interface AlertBannerProps { export const AlertBanner: React.FC = ({ color, message }) => { const bannerStyle = classnames(styles.alertBannerContainer, { [styles.default]: color === "default" || !color, - [styles.red]: color === "warning", + [styles.yellow]: color === "warning", }); return
{message}
; diff --git a/airbyte-webapp/src/components/ui/Breadcrumbs/Breadcrumbs.tsx b/airbyte-webapp/src/components/ui/Breadcrumbs/Breadcrumbs.tsx index 9b039209b7e..9044ac468a2 100644 --- a/airbyte-webapp/src/components/ui/Breadcrumbs/Breadcrumbs.tsx +++ b/airbyte-webapp/src/components/ui/Breadcrumbs/Breadcrumbs.tsx @@ -18,9 +18,9 @@ export const Breadcrumbs: React.FC = ({ data }) => { return ( <> {data.length && ( - + {data.map((item, index) => ( - + {item.to ? ( diff --git a/airbyte-webapp/src/components/ui/CodeEditor/CodeEditor.tsx b/airbyte-webapp/src/components/ui/CodeEditor/CodeEditor.tsx index 935eb89eda7..a95f6341c67 100644 --- a/airbyte-webapp/src/components/ui/CodeEditor/CodeEditor.tsx +++ b/airbyte-webapp/src/components/ui/CodeEditor/CodeEditor.tsx @@ -117,6 +117,7 @@ export const CodeEditor: React.FC = ({ top: paddingTopValue, } : {}, + fixedOverflowWidgets: true, }} /> ); diff --git a/airbyte-webapp/src/components/ui/DropdownMenu/DropdownMenu.module.scss b/airbyte-webapp/src/components/ui/DropdownMenu/DropdownMenu.module.scss index 56a43269d6a..a76859629da 100644 --- a/airbyte-webapp/src/components/ui/DropdownMenu/DropdownMenu.module.scss +++ b/airbyte-webapp/src/components/ui/DropdownMenu/DropdownMenu.module.scss @@ -6,7 +6,6 @@ z-index: z-indices.$dropdownMenu; overflow: auto; max-height: 300px; - padding: variables.$spacing-sm; outline: none; border-radius: variables.$border-radius-md; background-color: colors.$foreground; @@ -28,11 +27,10 @@ align-items: center; height: 42px; width: 100%; - padding: 0 variables.$spacing-lg; + padding: 0 variables.$spacing-xl; border: 0; background-color: transparent; text-decoration: none; - border-radius: variables.$border-radius-md; .icon { display: flex; @@ -52,7 +50,7 @@ } &.active { - background-color: colors.$grey-100; + background-color: colors.$blue-50; } &.disabled { diff --git a/airbyte-webapp/src/components/ui/Modal/Modal.tsx b/airbyte-webapp/src/components/ui/Modal/Modal.tsx index 85d80cf2174..d54f0b75542 100644 --- a/airbyte-webapp/src/components/ui/Modal/Modal.tsx +++ b/airbyte-webapp/src/components/ui/Modal/Modal.tsx @@ -13,11 +13,7 @@ import { Overlay } from "../Overlay"; export interface ModalProps { title?: string | React.ReactNode; /** - * Function to call when the user clicks on the close button (cross icon). - */ - onClose?: (reason: string) => void; - /** - * Function to call when the user clicks on overlay or press escape. + * Function to call when the user press Escape, clicks on Backdrop clicks or X-button clicks. * Note: if openModal function was called with "preventCancel: true" then this function will not be called. */ onCancel?: () => void; @@ -42,7 +38,6 @@ export const Modal: React.FC> = ({ children, title, size, - onClose, onCancel, cardless, testId, @@ -58,11 +53,6 @@ export const Modal: React.FC> = ({ } }; - const onModalClose = () => { - setIsOpen(false); - onClose?.("closeButtonClicked"); - }; - const Wrapper = wrapIn || "div"; return ( @@ -85,13 +75,15 @@ export const Modal: React.FC> = ({ {title} - + {onCancel && ( + + )}
{children} diff --git a/airbyte-webapp/src/components/ui/Separator/Separator.module.scss b/airbyte-webapp/src/components/ui/Separator/Separator.module.scss new file mode 100644 index 00000000000..ec683c41765 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Separator/Separator.module.scss @@ -0,0 +1,9 @@ +@use "scss/colors"; +@use "scss/variables"; + +.separator { + margin: 0 auto; + border: none; + width: 100%; + border-bottom: variables.$border-thin solid colors.$grey-100; +} \ No newline at end of file diff --git a/airbyte-webapp/src/components/ui/Separator/Separator.tsx b/airbyte-webapp/src/components/ui/Separator/Separator.tsx new file mode 100644 index 00000000000..4ef36511305 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Separator/Separator.tsx @@ -0,0 +1,11 @@ +import classNames from "classnames"; + +import styles from "./Separator.module.scss"; + +interface SeparatorProps { + className?: string; +} + +export const Separator: React.FC = ({ className }) => { + return
; +}; diff --git a/airbyte-webapp/src/components/ui/Separator/index.ts b/airbyte-webapp/src/components/ui/Separator/index.ts new file mode 100644 index 00000000000..0c664312946 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Separator/index.ts @@ -0,0 +1 @@ +export { Separator } from "./Separator"; diff --git a/airbyte-webapp/src/components/ui/Table/Table.module.scss b/airbyte-webapp/src/components/ui/Table/Table.module.scss index a0523e31219..30b6dcbc994 100644 --- a/airbyte-webapp/src/components/ui/Table/Table.module.scss +++ b/airbyte-webapp/src/components/ui/Table/Table.module.scss @@ -16,7 +16,7 @@ $border-radius: variables.$border-radius-lg; } } -.thead { +.thead--sticky { position: sticky; top: 0; z-index: z-indices.$tableScroll; diff --git a/airbyte-webapp/src/components/ui/Table/Table.tsx b/airbyte-webapp/src/components/ui/Table/Table.tsx index 6ad70332a31..42441be2260 100644 --- a/airbyte-webapp/src/components/ui/Table/Table.tsx +++ b/airbyte-webapp/src/components/ui/Table/Table.tsx @@ -30,6 +30,7 @@ export interface TableProps { testId?: string; columnVisibility?: VisibilityState; sorting?: boolean; + stickyHeaders?: boolean; getRowClassName?: (data: T) => string | undefined; initialSortBy?: Array<{ id: string; desc: boolean }>; /** @@ -56,6 +57,7 @@ export const Table = ({ expandedRow, columnVisibility, getRowClassName, + stickyHeaders = true, sorting = true, initialSortBy, virtualized = false, @@ -89,7 +91,7 @@ export const Table = ({ ); const TableHead: TableComponents["TableHead"] = React.forwardRef((props, ref) => ( - + )); TableHead.displayName = "TableHead"; @@ -210,7 +212,7 @@ export const Table = ({ })} data-testid={testId} > - {headerContent()} + {headerContent()} {rows.map((row) => ( diff --git a/airbyte-webapp/src/core/api/apiCall.ts b/airbyte-webapp/src/core/api/apiCall.ts index bea270c3eba..5e645612978 100644 --- a/airbyte-webapp/src/core/api/apiCall.ts +++ b/airbyte-webapp/src/core/api/apiCall.ts @@ -2,7 +2,6 @@ import { trackError } from "core/utils/datadog"; import { shortUuid } from "core/utils/uuid"; import { CommonRequestError } from "./errors/CommonRequestError"; -import { VersionError } from "./errors/VersionError"; export interface ApiCallOptions { getAccessToken: () => Promise; @@ -82,13 +81,7 @@ async function parseResponse(response: Response, requestUrl: string, response } if (response.headers.get("content-type") === "application/json") { - const jsonError = await response.json(); - - if (jsonError?.error?.startsWith("Version mismatch between")) { - throw new VersionError(jsonError.error); - } - - throw new CommonRequestError(response, jsonError); + throw new CommonRequestError(response, await response.json()); } let responseText: string | undefined; diff --git a/airbyte-webapp/src/core/api/errors/ErrorWithJobInfo.ts b/airbyte-webapp/src/core/api/errors/ErrorWithJobInfo.ts new file mode 100644 index 00000000000..c2cb21601c7 --- /dev/null +++ b/airbyte-webapp/src/core/api/errors/ErrorWithJobInfo.ts @@ -0,0 +1,21 @@ +import { SynchronousJobRead } from "../types/AirbyteClient"; + +/** + * An error that is linked to a synchronous job that ran (e.g. connector configuration check or discover schema) + * and has the job information attached to it. + */ +export class ErrorWithJobInfo extends Error { + constructor( + message: string, + public readonly jobInfo: SynchronousJobRead + ) { + super(message); + } + + /** + * Extract the job info from an error in case it is an ErrorWithJobInfo. + */ + static getJobInfo(error: Error | null): SynchronousJobRead | null { + return error instanceof ErrorWithJobInfo ? error.jobInfo : null; + } +} diff --git a/airbyte-webapp/src/core/api/errors/LogsRequestError.ts b/airbyte-webapp/src/core/api/errors/LogsRequestError.ts deleted file mode 100644 index 0e6f5f08b34..00000000000 --- a/airbyte-webapp/src/core/api/errors/LogsRequestError.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { SynchronousJobRead } from "core/api/types/AirbyteClient"; - -import { CommonRequestError } from "./CommonRequestError"; - -export class LogsRequestError extends CommonRequestError { - __type = "common.errorWithLogs"; - - constructor( - private jobInfo: SynchronousJobRead, - msg?: string - ) { - super(undefined, { message: msg }); - this._status = 400; - } - - static extractJobInfo(error: unknown) { - if (!error) { - return null; - } - return isLogsRequestError(error) ? error.jobInfo : null; - } -} - -export function isLogsRequestError(error: unknown): error is LogsRequestError { - return (error as LogsRequestError).__type === "common.errorWithLogs"; -} diff --git a/airbyte-webapp/src/core/api/errors/ServerError.ts b/airbyte-webapp/src/core/api/errors/ServerError.ts deleted file mode 100644 index cc0e2d7e443..00000000000 --- a/airbyte-webapp/src/core/api/errors/ServerError.ts +++ /dev/null @@ -1,3 +0,0 @@ -export abstract class ServerError extends Error { - __type = "unknown.error"; -} diff --git a/airbyte-webapp/src/core/api/errors/VersionError.ts b/airbyte-webapp/src/core/api/errors/VersionError.ts deleted file mode 100644 index 563f775773d..00000000000 --- a/airbyte-webapp/src/core/api/errors/VersionError.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { ServerError } from "./ServerError"; - -export class VersionError extends ServerError { - __type = "version.mismatch"; -} - -export function isVersionError(error: { __type?: string }): error is VersionError { - return error.__type === "version.mismatch"; -} diff --git a/airbyte-webapp/src/core/api/errors/index.ts b/airbyte-webapp/src/core/api/errors/index.ts index 36fa2d27bbc..b7693522111 100644 --- a/airbyte-webapp/src/core/api/errors/index.ts +++ b/airbyte-webapp/src/core/api/errors/index.ts @@ -1,4 +1,2 @@ export * from "./CommonRequestError"; -export * from "./VersionError"; -export * from "./LogsRequestError"; -export * from "./ServerError"; +export * from "./ErrorWithJobInfo"; diff --git a/airbyte-webapp/src/core/api/hooks/cloud/users.ts b/airbyte-webapp/src/core/api/hooks/cloud/users.ts index ce17554331c..d0285829dbd 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/users.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/users.ts @@ -1,9 +1,12 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useCallback, useMemo } from "react"; +import { useIntl } from "react-intl"; import { useCurrentWorkspaceId } from "area/workspace/utils"; +import { useAuthService, useCurrentUser } from "core/services/auth"; import { trackAction } from "core/utils/datadog"; import { AppActionCodes } from "hooks/services/AppMonitoringService"; +import { useNotificationService } from "hooks/services/Notification"; import { webBackendRevokeUserFromWorkspace, @@ -13,6 +16,7 @@ import { updateUser, webBackendRevokeUserSession, createKeycloakUser, + sendVerificationEmail, } from "../../generated/CloudApi"; import { SCOPE_WORKSPACE } from "../../scopes"; import { CreateKeycloakUserRequestBody, UserUpdate } from "../../types/CloudApi"; @@ -175,3 +179,38 @@ export const useCreateKeycloakUser = () => { }) ); }; + +const RESEND_EMAIL_TOAST_ID = "resendEmail"; +export const useResendEmailVerification = () => { + const { userId } = useCurrentUser(); + const requestOptions = useRequestOptions(); + const { sendEmailVerification } = useAuthService(); + const { registerNotification } = useNotificationService(); + const { formatMessage } = useIntl(); + + const sendEmail = useMemo(() => { + // The old way to send a verification email with Firebase, via the AuthContext + if (sendEmailVerification) { + return sendEmailVerification; + } + // The new way to send a verification email with Keycloak via our API + return () => sendVerificationEmail({ userId }, requestOptions); + }, [sendEmailVerification, userId, requestOptions]); + + return useMutation(sendEmail, { + onSuccess: () => { + registerNotification({ + id: RESEND_EMAIL_TOAST_ID, + type: "success", + text: formatMessage({ id: "credits.emailVerification.resendConfirmation" }), + }); + }, + onError: () => { + registerNotification({ + id: RESEND_EMAIL_TOAST_ID, + type: "error", + text: formatMessage({ id: "credits.emailVerification.resendConfirmationError" }), + }); + }, + }); +}; diff --git a/airbyte-webapp/src/core/api/hooks/connections.tsx b/airbyte-webapp/src/core/api/hooks/connections.tsx index 89c7bb089e4..5b938166897 100644 --- a/airbyte-webapp/src/core/api/hooks/connections.tsx +++ b/airbyte-webapp/src/core/api/hooks/connections.tsx @@ -6,6 +6,7 @@ import { useNavigate } from "react-router-dom"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { getFrequencyFromScheduleData, useAnalyticsService, Action, Namespace } from "core/services/analytics"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; +import { useExperiment } from "hooks/services/Experiment"; import { useNotificationService } from "hooks/services/Notification"; import { CloudRoutes } from "packages/cloud/cloudRoutePaths"; import { RoutePaths } from "pages/routePaths"; @@ -218,6 +219,7 @@ export const useCreateConnection = () => { const queryClient = useQueryClient(); const analyticsService = useAnalyticsService(); const invalidateWorkspaceSummary = useInvalidateWorkspaceStateQuery(); + const isSimplifiedCreation = useExperiment("connection.simplifiedCreation", false); return useMutation( async ({ @@ -253,6 +255,8 @@ export const useCreateConnection = () => { available_streams: values.syncCatalog.streams.length, enabled_streams: enabledStreams.length, enabled_streams_list: JSON.stringify(enabledStreams), + connection_id: response.connectionId, + is_simplified_creation: isSimplifiedCreation, }); return response; diff --git a/airbyte-webapp/src/core/api/hooks/connectorBuilderApi.ts b/airbyte-webapp/src/core/api/hooks/connectorBuilderApi.ts index 580bba2b5ba..df59792fc0d 100644 --- a/airbyte-webapp/src/core/api/hooks/connectorBuilderApi.ts +++ b/airbyte-webapp/src/core/api/hooks/connectorBuilderApi.ts @@ -1,6 +1,6 @@ import { useQuery } from "@tanstack/react-query"; -import { DEFAULT_JSON_MANIFEST_VALUES } from "components/connectorBuilder/types"; +import { DEFAULT_JSON_MANIFEST_VALUES, ManifestValuePerComponentPerStream } from "components/connectorBuilder/types"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { CommonRequestError } from "core/api/errors"; @@ -26,7 +26,9 @@ const connectorBuilderKeys = { list: (manifest: ConnectorManifest, config: ConnectorConfig) => [...connectorBuilderKeys.all, "list", { manifest, config }] as const, template: ["template"] as const, - resolve: (manifest?: ConnectorManifest) => [...connectorBuilderKeys.all, "resolve", { manifest }] as const, + resolveYaml: (manifest?: ConnectorManifest) => [...connectorBuilderKeys.all, "resolve", { manifest }] as const, + resolveUi: (manifestValuePerComponentPerStream: ManifestValuePerComponentPerStream) => + [...connectorBuilderKeys.all, "resolve", manifestValuePerComponentPerStream] as const, resolveSuspense: (manifest?: ConnectorManifest) => [...connectorBuilderKeys.all, "resolveSuspense", { manifest }] as const, }; @@ -45,11 +47,17 @@ export const useBuilderReadStream = ( }); }; -export const useBuilderResolvedManifest = (params: ResolveManifestRequestBody, enabled = true) => { +export const useBuilderResolvedManifest = ( + params: ResolveManifestRequestBody, + enabled = true, + manifestValuePerComponentPerStream?: ManifestValuePerComponentPerStream +) => { const requestOptions = useRequestOptions(); return useQuery>( - connectorBuilderKeys.resolve(params.manifest), + manifestValuePerComponentPerStream === undefined + ? connectorBuilderKeys.resolveYaml(params.manifest) + : connectorBuilderKeys.resolveUi(manifestValuePerComponentPerStream), () => resolveManifest(params, requestOptions), { keepPreviousData: true, diff --git a/airbyte-webapp/src/core/api/hooks/connectorCheck.ts b/airbyte-webapp/src/core/api/hooks/connectorCheck.ts index 3c01a335f9e..81c4591bd39 100644 --- a/airbyte-webapp/src/core/api/hooks/connectorCheck.ts +++ b/airbyte-webapp/src/core/api/hooks/connectorCheck.ts @@ -1,7 +1,8 @@ import { useMutation } from "@tanstack/react-query"; +import { useIntl } from "react-intl"; import { ConnectionConfiguration } from "area/connector/types"; -import { LogsRequestError } from "core/api"; +import { ErrorWithJobInfo } from "core/api"; import { checkConnectionToDestination, @@ -41,6 +42,7 @@ export type ConnectorCheckParams = CreateConnectorParams | RetestConnectorParams export const useCheckConnector = (type: "source" | "destination") => { const requestOptions = useRequestOptions(); + const { formatMessage } = useIntl(); return useMutation(async (params) => { const options = { ...requestOptions, signal: params.signal }; let result: CheckConnectionRead; @@ -101,9 +103,9 @@ export const useCheckConnector = (type: "source" | "destination") => { } if (!result.jobInfo?.succeeded) { - throw new LogsRequestError(result.jobInfo, "Failed to run connection tests."); + throw new ErrorWithJobInfo(formatMessage({ id: "connector.check.jobFailed" }), result.jobInfo); } else if (result.status === CheckConnectionReadStatus.failed) { - throw new LogsRequestError(result.jobInfo, result.message); + throw new ErrorWithJobInfo(result.message ?? formatMessage({ id: "connector.check.failed" }), result.jobInfo); } return result; diff --git a/airbyte-webapp/src/core/api/hooks/sources.tsx b/airbyte-webapp/src/core/api/hooks/sources.tsx index 41b94e8d199..99a2df1715c 100644 --- a/airbyte-webapp/src/core/api/hooks/sources.tsx +++ b/airbyte-webapp/src/core/api/hooks/sources.tsx @@ -1,6 +1,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useCallback, useEffect, useState } from "react"; import { flushSync } from "react-dom"; +import { useIntl } from "react-intl"; import { ConnectionConfiguration } from "area/connector/types"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; @@ -8,7 +9,7 @@ import { isDefined } from "core/utils/common"; import { useRemoveConnectionsFromList } from "./connections"; import { useCurrentWorkspace } from "./workspaces"; -import { CommonRequestError, LogsRequestError } from "../errors"; +import { ErrorWithJobInfo } from "../errors"; import { createSource, deleteSource, @@ -18,7 +19,7 @@ import { updateSource, } from "../generated/AirbyteClient"; import { SCOPE_WORKSPACE } from "../scopes"; -import { AirbyteCatalog, SourceRead, SynchronousJobRead, WebBackendConnectionListItem } from "../types/AirbyteClient"; +import { AirbyteCatalog, SourceRead, WebBackendConnectionListItem } from "../types/AirbyteClient"; import { useRequestErrorHandler } from "../useRequestErrorHandler"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; @@ -174,23 +175,22 @@ const useUpdateSource = () => { ); }; -export type SchemaError = (Error & { status: number; response: SynchronousJobRead }) | null; - const useDiscoverSchema = ( sourceId: string, disableCache?: boolean ): { isLoading: boolean; schema: AirbyteCatalog | undefined; - schemaErrorStatus: SchemaError; + schemaErrorStatus: Error | null; catalogId: string | undefined; onDiscoverSchema: () => Promise; } => { + const { formatMessage } = useIntl(); const requestOptions = useRequestOptions(); const [schema, setSchema] = useState(undefined); const [catalogId, setCatalogId] = useState(""); const [isLoading, setIsLoading] = useState(false); - const [schemaErrorStatus, setSchemaErrorStatus] = useState(null); + const [schemaErrorStatus, setSchemaErrorStatus] = useState(null); const onDiscoverSchema = useCallback(async () => { setIsLoading(true); @@ -201,14 +201,11 @@ const useDiscoverSchema = ( requestOptions ); - if (!result.jobInfo?.succeeded || !result.catalog) { - // @ts-expect-error TODO: address this case - const e = result.jobInfo?.logs ? new LogsRequestError(result.jobInfo) : new CommonRequestError(result); - // Generate error with failed status and received logs - e._status = 400; - // @ts-expect-error TODO: address this case - e.response = result.jobInfo; - throw e; + if (!result.jobInfo?.succeeded) { + throw new ErrorWithJobInfo(formatMessage({ id: "connector.discoverSchema.jobFailed" }), result.jobInfo); + } + if (!result.catalog) { + throw new ErrorWithJobInfo(formatMessage({ id: "connector.discoverSchema.catalogMissing" }), result.jobInfo); } flushSync(() => { @@ -220,7 +217,7 @@ const useDiscoverSchema = ( } finally { setIsLoading(false); } - }, [disableCache, requestOptions, sourceId]); + }, [disableCache, formatMessage, requestOptions, sourceId]); useEffect(() => { if (sourceId) { diff --git a/airbyte-webapp/src/core/api/hooks/userInvitations.tsx b/airbyte-webapp/src/core/api/hooks/userInvitations.tsx index 1de4215e630..ca78728aef5 100644 --- a/airbyte-webapp/src/core/api/hooks/userInvitations.tsx +++ b/airbyte-webapp/src/core/api/hooks/userInvitations.tsx @@ -1,12 +1,20 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useIntl } from "react-intl"; -import { useNotificationService } from "hooks/services/Notification"; +import { MessageType } from "components/ui/Message"; + +import { Notification, useNotificationService } from "hooks/services/Notification"; import { workspaceKeys } from "./workspaces"; -import { acceptUserInvitation, createUserInvitation, listPendingInvitations } from "../generated/AirbyteClient"; +import { + acceptUserInvitation, + createUserInvitation, + listPendingInvitations, + cancelUserInvitation, +} from "../generated/AirbyteClient"; import { SCOPE_ORGANIZATION, SCOPE_USER, SCOPE_WORKSPACE } from "../scopes"; import { + InviteCodeRequestBody, UserInvitationCreateRequestBody, UserInvitationListRequestBody, UserInvitationRead, @@ -33,12 +41,29 @@ export const useAcceptUserInvitation = (inviteCode?: string | null): UserInvitat queryClient.invalidateQueries(workspaceKeys.lists()); return response; }) - .catch(() => { - registerNotification({ - type: "error", - text: formatMessage({ id: "userInvitations.accept.error" }), - id: "userInvitations.accept.error", - }); + .catch((err: { message: string; status?: number }) => { + const getNotificationFromError = (err: { message: string; status?: number }): Notification => { + let notificationId = "userInvitations.accept.error"; + let notificationType: MessageType = "error"; + + if (err.status === 403) { + notificationId = "userInvitations.accept.error.email"; + } else if (err.message.endsWith("Status: expired")) { + notificationId = "userInvitations.accept.error.expired"; + } else if (err.message.endsWith("Status: cancelled")) { + notificationId = "userInvitations.accept.error.cancelled"; + } else if (err.message.endsWith("Status: accepted")) { + notificationId = "userInvitations.accept.warning.alreadyAccepted"; + notificationType = "info"; + } + + return { + type: notificationType, + text: formatMessage({ id: notificationId }), + id: notificationId, + }; + }; + registerNotification(getNotificationFromError(err)); return null; }), { enabled: !!inviteCode } @@ -59,10 +84,23 @@ export const useCreateUserInvitation = () => { text: formatMessage({ id: "userInvitations.create.success" }), id: "userInvitations.create.success", }); + const keyScope = invitationCreate.scopeType === "workspace" ? SCOPE_WORKSPACE : SCOPE_ORGANIZATION; + + // this endpoint will direct add users who are already within the org, so we want to invalidate both the invitations and the members lists queryClient.invalidateQueries(workspaceKeys.allListAccessUsers); + queryClient.invalidateQueries([keyScope, "userInvitations"]); return response; }) - .catch(() => { + .catch((err) => { + if (err.status === 409) { + registerNotification({ + type: "error", + text: formatMessage({ id: "userInvitations.create.error.duplicate" }), + id: "userInvitations.create.error.duplicate", + }); + return null; + } + registerNotification({ type: "error", text: formatMessage({ id: "userInvitations.create.error" }), @@ -80,3 +118,34 @@ export const useListUserInvitations = (userInvitationListRequestBody: UserInvita listPendingInvitations(userInvitationListRequestBody, requestOptions) ); }; + +export const useCancelUserInvitation = () => { + const requestOptions = useRequestOptions(); + const queryClient = useQueryClient(); + const { formatMessage } = useIntl(); + const { registerNotification } = useNotificationService(); + + return useMutation(async (inviteCodeRequestBody: InviteCodeRequestBody) => + cancelUserInvitation(inviteCodeRequestBody, requestOptions) + .then((res) => { + registerNotification({ + type: "success", + text: formatMessage({ id: "userInvitations.cancel.success" }), + id: "userInvitations.cancel.success", + }); + + queryClient.invalidateQueries([ + res.scopeType === "organization" ? SCOPE_ORGANIZATION : SCOPE_WORKSPACE, + "userInvitations", + ]); + }) + .catch(() => { + registerNotification({ + type: "error", + text: formatMessage({ id: "userInvitations.cancel.error" }), + id: "userInvitations.cancel.error", + }); + return null; + }) + ); +}; diff --git a/airbyte-webapp/src/core/api/types/.gitignore b/airbyte-webapp/src/core/api/types/.gitignore index d431763a5b4..71be0286750 100644 --- a/airbyte-webapp/src/core/api/types/.gitignore +++ b/airbyte-webapp/src/core/api/types/.gitignore @@ -1,5 +1,2 @@ # Ignore all type export files, since they are generated by orval *.ts - -# Allow checking in manual generated files -!AirbyteApi.ts \ No newline at end of file diff --git a/airbyte-webapp/src/core/api/types/AirbyteApi.ts b/airbyte-webapp/src/core/api/types/AirbyteApi.ts deleted file mode 100644 index a1a618e827d..00000000000 --- a/airbyte-webapp/src/core/api/types/AirbyteApi.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "../generated/AirbyteApi.schemas"; diff --git a/airbyte-webapp/src/core/config/ConfigServiceProvider.tsx b/airbyte-webapp/src/core/config/ConfigServiceProvider.tsx deleted file mode 100644 index 4b0282e48a7..00000000000 --- a/airbyte-webapp/src/core/config/ConfigServiceProvider.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import React, { useContext } from "react"; - -import { AirbyteWebappConfig } from "./types"; - -export interface ConfigContextData { - config: AirbyteWebappConfig; -} - -export const ConfigContext = React.createContext(null); - -export function useConfig(): AirbyteWebappConfig { - const configService = useContext(ConfigContext); - - if (configService === null) { - throw new Error("useConfig must be used within a ConfigProvider"); - } - - return configService.config; -} - -export const ConfigServiceProvider: React.FC< - React.PropsWithChildren<{ - config: AirbyteWebappConfig; - }> -> = ({ children, config }) => { - return {children}; -}; diff --git a/airbyte-webapp/src/core/config/index.ts b/airbyte-webapp/src/core/config/index.ts index 87a0212e66d..5c62e04f5ee 100644 --- a/airbyte-webapp/src/core/config/index.ts +++ b/airbyte-webapp/src/core/config/index.ts @@ -1,3 +1 @@ export * from "./config"; -export * from "./ConfigServiceProvider"; -export * from "./types"; diff --git a/airbyte-webapp/src/core/services/analytics/pageTrackingCodes.tsx b/airbyte-webapp/src/core/services/analytics/pageTrackingCodes.tsx index 1120516a9c6..595b4d2d846 100644 --- a/airbyte-webapp/src/core/services/analytics/pageTrackingCodes.tsx +++ b/airbyte-webapp/src/core/services/analytics/pageTrackingCodes.tsx @@ -12,6 +12,10 @@ export enum PageTrackingCodes { DESTINATION_ITEM = "Destination.Item", DESTINATION_ITEM_SETTINGS = "Destination.Item.Settings", CONNECTIONS_NEW = "Connections.New", + CONNECTIONS_NEW_DEFINE_SOURCE = "Connections.New.DefineSource", + CONNECTIONS_NEW_DEFINE_DESTINATION = "Connections.New.DefineDestination", + CONNECTIONS_NEW_SELECT_STREAMS = "Connections.New.SelectStreams", + CONNECTIONS_NEW_CONFIGURE_CONNECTION = "Connections.New.ConfigureConnection", CONNECTIONS_LIST = "Connections.List", CONNECTIONS_ITEM = "Connections.Item", CONNECTIONS_ITEM_STATUS = "Connections.Item.Status", diff --git a/airbyte-webapp/src/core/services/analytics/types.ts b/airbyte-webapp/src/core/services/analytics/types.ts index fc56fe82fc9..f514ccf1d8c 100644 --- a/airbyte-webapp/src/core/services/analytics/types.ts +++ b/airbyte-webapp/src/core/services/analytics/types.ts @@ -10,6 +10,9 @@ export const enum Namespace { SCHEMA = "Schema", ERD = "ERD", SETTINGS = "Settings", + SYNC_QUESTIONNAIRE = "SyncQuestionnaire", + STREAM_SELECTION = "StreamSelection", + FORM = "Form", } export const enum Action { @@ -39,6 +42,11 @@ export const enum Action { DOWNLOAD_SCHEDULER_LOGS = "DownloadSchedulerLogs", UPGRADE_VERSION = "UpgradeVersion", DISCOVER_SCHEMA = "DiscoverSchema", + DISPLAYED = "Displayed", + ANSWERED = "Answered", + APPLIED = "Applied", + SET_SYNC_MODE = "SetSyncMode", + DISMISSED_CHANGES_MODAL = "DismissedChangesModal", // Connector Builder Actions CONNECTOR_BUILDER_START = "ConnectorBuilderStart", diff --git a/airbyte-webapp/src/core/services/auth/AuthContext.ts b/airbyte-webapp/src/core/services/auth/AuthContext.ts index 8077eaeb520..2cf457058ad 100644 --- a/airbyte-webapp/src/core/services/auth/AuthContext.ts +++ b/airbyte-webapp/src/core/services/auth/AuthContext.ts @@ -17,7 +17,9 @@ export type AuthSignUp = (form: SignupFormValues) => Promise; export type AuthChangeName = (name: string) => Promise; export type AuthSendEmailVerification = () => Promise; -export type AuthVerifyEmail = (code: string) => Promise; +export type AuthVerifyEmail = FirebaseVerifyEmail | KeycloakVerifyEmail; +type FirebaseVerifyEmail = (code: string) => Promise; +type KeycloakVerifyEmail = () => Promise; export type AuthLogout = () => Promise; export type OAuthLoginState = "waiting" | "loading" | "done"; diff --git a/airbyte-webapp/src/core/utils/links.ts b/airbyte-webapp/src/core/utils/links.ts index ffe31650ff8..d3d87c15b53 100644 --- a/airbyte-webapp/src/core/utils/links.ts +++ b/airbyte-webapp/src/core/utils/links.ts @@ -30,7 +30,7 @@ export const links = { webpageLink: "https://airbyte.com", webhookVideoGuideLink: "https://www.youtube.com/watch?v=NjYm8F-KiFc", cronReferenceLink: "http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html", - cloudAllowlistIPsLink: `${BASE_DOCS_LINK}/cloud/getting-started-with-airbyte-cloud/#allowlist-ip-addresses`, + cloudAllowlistIPsLink: `${BASE_DOCS_LINK}/operating-airbyte/security#network-security-1`, dataResidencySurvey: "https://forms.gle/Dr7MPTdt9k3xTinL8", connectionDataResidency: `${BASE_DOCS_LINK}/cloud/managing-airbyte-cloud/manage-data-residency#choose-the-data-residency-for-a-connection`, lowCodeYamlDescription: `${BASE_DOCS_LINK}/connector-development/config-based/understanding-the-yaml-file/yaml-overview`, diff --git a/airbyte-webapp/src/core/utils/pollUntil.test.ts b/airbyte-webapp/src/core/utils/pollUntil.test.ts deleted file mode 100644 index 9ecdc97fdc2..00000000000 --- a/airbyte-webapp/src/core/utils/pollUntil.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { pollUntil } from "./pollUntil"; - -// a toy promise that can be polled for a specific response -const fourZerosAndThenSeven = () => { - let _callCount = 0; - return () => Promise.resolve([0, 0, 0, 0, 7][_callCount++]); -}; - -const truthyResponse = (x: unknown) => !!x; - -describe("pollUntil", () => { - beforeAll(() => { - jest.useFakeTimers({ doNotFake: ["nextTick"] }); - }); - - afterAll(() => { - jest.useRealTimers(); - }); - - describe("when maxTimeoutMs is not provided", () => { - it("calls the provided apiFn until condition returns true and resolves to its final return value", () => { - const pollableFn = fourZerosAndThenSeven(); - const result = pollUntil(pollableFn, truthyResponse, { intervalMs: 1 }); - jest.advanceTimersByTime(10); - return expect(result).resolves.toBe(7); - }); - }); - - describe("when condition returns true before maxTimeoutMs is reached", () => { - it("calls the provided apiFn until condition returns true and resolves to its final return value", () => { - const pollableFn = fourZerosAndThenSeven(); - const result = pollUntil(pollableFn, truthyResponse, { intervalMs: 1, maxTimeoutMs: 100 }); - jest.advanceTimersByTime(10); - return expect(result).resolves.toBe(7); - }); - }); - - describe("when maxTimeoutMs is reached before condition returns true", () => { - it("resolves to false", () => { - const pollableFn = fourZerosAndThenSeven(); - const result = pollUntil(pollableFn, truthyResponse, { intervalMs: 100, maxTimeoutMs: 1 }); - jest.advanceTimersByTime(100); - return expect(result).resolves.toBe(false); - }); - - it("calls its apiFn arg no more than (maxTimeoutMs / intervalMs) times", async () => { - let _callCount = 0; - const pollableFn = jest.fn(() => { - return Promise.resolve([1, 2, 3, 4, 5][_callCount++]); - }); - - const polling = pollUntil(pollableFn, (_) => false, { intervalMs: 20, maxTimeoutMs: 78 }); - - // Advance the timer by 20ms each. Make sure to wait one more tick (which isn't using fake timers) - // so rxjs will actually call pollableFn again (and thus we get the right count on the that function). - // Without waiting a tick after each advance timer, we'd effectively just advance by 80ms and - // not call the pollableFn multiple times, because the maxTimeout logic would be triggered which - // would cause the subsequent pollableFn calls to not be properly processed. - jest.advanceTimersByTime(20); - await new Promise(process.nextTick); - jest.advanceTimersByTime(20); - await new Promise(process.nextTick); - jest.advanceTimersByTime(20); - await new Promise(process.nextTick); - jest.advanceTimersByTime(20); - await new Promise(process.nextTick); - - const result = await polling; - - expect(result).toBe(false); - expect(pollableFn).toHaveBeenCalledTimes(4); - }); - }); -}); diff --git a/airbyte-webapp/src/core/utils/pollUntil.ts b/airbyte-webapp/src/core/utils/pollUntil.ts deleted file mode 100644 index 7349cf0459e..00000000000 --- a/airbyte-webapp/src/core/utils/pollUntil.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { timer, delay, from, concatMap, takeWhile, last, raceWith, lastValueFrom, NEVER } from "rxjs"; - -// Known issues: -// - the case where `apiFn` returns `false` and `condition(false) === true` is impossible to distinguish from a timeout -export function pollUntil( - apiFn: () => Promise, - condition: (res: ResponseType) => boolean, - options: { intervalMs: number; maxTimeoutMs?: number } -) { - const { intervalMs, maxTimeoutMs } = options; - const poll$ = timer(0, intervalMs).pipe( - concatMap(() => from(apiFn())), - takeWhile((result) => !condition(result), true), - last() - ); - - const timeout$ = maxTimeoutMs ? from([false]).pipe(delay(maxTimeoutMs)) : NEVER; - - return lastValueFrom(poll$.pipe(raceWith(timeout$))); -} diff --git a/airbyte-webapp/src/hooks/connection/useConfirmCatalogDiff.tsx b/airbyte-webapp/src/hooks/connection/useConfirmCatalogDiff.tsx index 3fc149c8383..fd0daee38ed 100644 --- a/airbyte-webapp/src/hooks/connection/useConfirmCatalogDiff.tsx +++ b/airbyte-webapp/src/hooks/connection/useConfirmCatalogDiff.tsx @@ -1,7 +1,7 @@ import { useEffect } from "react"; import { useIntl } from "react-intl"; -import { CatalogDiffModal } from "components/connection/CatalogDiffModal/CatalogDiffModal"; +import { CatalogDiffModal } from "components/connection/CatalogDiffModal"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { useModalService } from "hooks/services/Modal"; @@ -20,8 +20,8 @@ export const useConfirmCatalogDiff = () => { preventCancel: true, size: "md", testId: "catalog-diff-modal", - content: ({ onClose }) => ( - + content: ({ onComplete }) => ( + ), }); } diff --git a/airbyte-webapp/src/hooks/services/ConfirmationModal/ConfirmationModalService.tsx b/airbyte-webapp/src/hooks/services/ConfirmationModal/ConfirmationModalService.tsx index 2b13d0c2bc5..b758c1fbfcf 100644 --- a/airbyte-webapp/src/hooks/services/ConfirmationModal/ConfirmationModalService.tsx +++ b/airbyte-webapp/src/hooks/services/ConfirmationModal/ConfirmationModalService.tsx @@ -4,7 +4,7 @@ import { ConfirmationModal } from "components/common/ConfirmationModal"; import useTypesafeReducer from "hooks/useTypesafeReducer"; -import { actions, initialState, confirmationModalServiceReducer } from "./reducer"; +import { actions, confirmationModalServiceReducer, initialState } from "./reducer"; import { ConfirmationModalOptions, ConfirmationModalServiceApi, ConfirmationModalState } from "./types"; const ConfirmationModalServiceContext = React.createContext(undefined); @@ -52,9 +52,9 @@ export const ConfirmationModalService = ({ children }: { children: React.ReactNo [closeConfirmationModal, openConfirmationModal] ); - const onClose = useCallback(() => { + const onCancel = useCallback(() => { closeConfirmationModal(); - state.confirmationModal?.onClose?.(); + state.confirmationModal?.onCancel?.(); }, [closeConfirmationModal, state.confirmationModal]); return ( @@ -64,11 +64,12 @@ export const ConfirmationModalService = ({ children }: { children: React.ReactNo {state.isOpen && state.confirmationModal ? ( & { - onClose?: ConfirmationModalProps["onClose"]; +export type ConfirmationModalOptions = Omit & { + onCancel?: ConfirmationModalProps["onCancel"]; }; export interface ConfirmationModalServiceApi { diff --git a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx index 39cc792d0fb..845ab2a529a 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx @@ -3,13 +3,7 @@ import { createContext, useCallback, useContext, useState } from "react"; import { useIntl } from "react-intl"; import { useAsyncFn } from "react-use"; -import { - SchemaError, - useCurrentWorkspace, - useGetConnection, - useGetConnectionQuery, - useUpdateConnection, -} from "core/api"; +import { useCurrentWorkspace, useGetConnection, useGetConnectionQuery, useUpdateConnection } from "core/api"; import { AirbyteCatalog, ConnectionStatus, @@ -170,7 +164,7 @@ export const ConnectionEditServiceProvider: React.FC {children} diff --git a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx index f68b1221daf..8f8fb17f453 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx @@ -11,7 +11,6 @@ import { useGetDestinationDefinitionSpecification, useSourceDefinition, useDestinationDefinition, - SchemaError, } from "core/api"; import { ActorDefinitionVersionRead, @@ -34,7 +33,7 @@ export type ConnectionOrPartialConnection = interface ConnectionServiceProps { connection: ConnectionOrPartialConnection; mode: ConnectionFormMode; - schemaError?: SchemaError | null; + schemaError?: Error | null; refreshSchema: () => Promise; } @@ -48,7 +47,7 @@ interface ConnectionFormHook { destDefinitionVersion: ActorDefinitionVersionRead; destDefinitionSpecification: DestinationDefinitionSpecificationRead; initialValues: FormConnectionFormValues; - schemaError?: SchemaError; + schemaError?: Error | null; refreshSchema: () => Promise; setSubmitError: (submitError: FormError | null) => void; getErrorMessage: (formValid: boolean, errors?: FieldErrors) => string | JSX.Element | null; diff --git a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts index dfda23da395..cb40c8b2e78 100644 --- a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts +++ b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts @@ -13,6 +13,7 @@ export interface Experiments { "billing.early-sync-enabled": boolean; "billing.autoRecharge": boolean; "connections.summaryView": boolean; + "connection.clearNotReset": boolean; "connection.columnSelection": boolean; "connection.simplifiedCreation": boolean; "connection.onboarding.destinations": string; @@ -21,7 +22,6 @@ export interface Experiments { "connection.streamCentricUI.lateMultiplier": number; "connection.streamCentricUI.v2": boolean; "connection.streamCentricUI.historicalOverview": boolean; - "connection.syncCatalog.simplifiedCatalogRow": boolean; "connector.airbyteCloudIpAddresses": string; "connector.suggestedSourceConnectors": string; "connector.suggestedDestinationConnectors": string; diff --git a/airbyte-webapp/src/hooks/services/FormChangeTracker/FormChangeTrackerService.tsx b/airbyte-webapp/src/hooks/services/FormChangeTracker/FormChangeTrackerService.tsx index 4e179b55ed0..449253aff82 100644 --- a/airbyte-webapp/src/hooks/services/FormChangeTracker/FormChangeTrackerService.tsx +++ b/airbyte-webapp/src/hooks/services/FormChangeTracker/FormChangeTrackerService.tsx @@ -1,14 +1,16 @@ import React, { useCallback } from "react"; +import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { Blocker, useBlocker } from "core/services/navigation"; -import { useFormChangeTrackerService } from "./hooks"; +import { isGeneratedFormId, useFormChangeTrackerService } from "./hooks"; import { useConfirmationModalService } from "../ConfirmationModal"; export const FormChangeTrackerService: React.FC> = ({ children }) => { - const { hasFormChanges, clearAllFormChanges } = useFormChangeTrackerService(); + const { hasFormChanges, clearAllFormChanges, getDirtyFormIds } = useFormChangeTrackerService(); const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); + const analyticsService = useAnalyticsService(); const blocker = useCallback( (blocker: Blocker) => { openConfirmationModal({ @@ -16,13 +18,21 @@ export const FormChangeTrackerService: React.FC text: "form.unsavedChangesMessage", submitButtonText: "form.leavePage", onSubmit: () => { + const dirtyFormIds = getDirtyFormIds().filter((id) => !isGeneratedFormId(id)); + if (dirtyFormIds.length) { + analyticsService.track(Namespace.FORM, Action.DISMISSED_CHANGES_MODAL, { + actionDescription: "User dismissed the leave changes modal", + dirtyFormIds, + }); + } + clearAllFormChanges(); closeConfirmationModal(); blocker.proceed(); }, }); }, - [clearAllFormChanges, closeConfirmationModal, openConfirmationModal] + [clearAllFormChanges, closeConfirmationModal, openConfirmationModal, getDirtyFormIds, analyticsService] ); useBlocker(blocker, hasFormChanges); diff --git a/airbyte-webapp/src/hooks/services/FormChangeTracker/hooks.ts b/airbyte-webapp/src/hooks/services/FormChangeTracker/hooks.ts index 370121592a9..0330d1ce173 100644 --- a/airbyte-webapp/src/hooks/services/FormChangeTracker/hooks.ts +++ b/airbyte-webapp/src/hooks/services/FormChangeTracker/hooks.ts @@ -7,7 +7,8 @@ import { FormChangeTrackerServiceApi } from "./types"; const changedForms = new Set(); const useHasFormChanges = createGlobalState(false); -export const useUniqueFormId = (formId?: string) => useMemo(() => formId ?? uniqueId("form_"), [formId]); +export const useUniqueFormId = (formId?: string) => useMemo(() => formId ?? uniqueId("unique_form_"), [formId]); +export const isGeneratedFormId = (id: string) => id.match(/^unique_form_\d+/); export const useFormChangeTrackerService = (): FormChangeTrackerServiceApi => { const [hasFormChanges, setHasFormChanges] = useHasFormChanges(); @@ -37,10 +38,13 @@ export const useFormChangeTrackerService = (): FormChangeTrackerServiceApi => { [setHasFormChanges] ); + const getDirtyFormIds = useCallback(() => Array.from(changedForms), []); + return { hasFormChanges, trackFormChange, clearFormChange, clearAllFormChanges, + getDirtyFormIds, }; }; diff --git a/airbyte-webapp/src/hooks/services/FormChangeTracker/types.ts b/airbyte-webapp/src/hooks/services/FormChangeTracker/types.ts index b14fc275f0e..1ce42189c00 100644 --- a/airbyte-webapp/src/hooks/services/FormChangeTracker/types.ts +++ b/airbyte-webapp/src/hooks/services/FormChangeTracker/types.ts @@ -3,4 +3,5 @@ export interface FormChangeTrackerServiceApi { trackFormChange: (id: string, changed: boolean) => void; clearFormChange: (id: string) => void; clearAllFormChanges: () => void; + getDirtyFormIds: () => string[]; } diff --git a/airbyte-webapp/src/hooks/services/Modal/ModalService.test.tsx b/airbyte-webapp/src/hooks/services/Modal/ModalService.test.tsx index 1c808e4c90b..4d7438a28e1 100644 --- a/airbyte-webapp/src/hooks/services/Modal/ModalService.test.tsx +++ b/airbyte-webapp/src/hooks/services/Modal/ModalService.test.tsx @@ -12,15 +12,15 @@ const TestComponent: React.FC<{ onModalResult?: (result: ModalResult) = useEffectOnce(() => { openModal({ title: "Test Modal Title", - content: ({ onCancel, onClose }) => ( + content: ({ onComplete, onCancel }) => (
- -
@@ -80,7 +80,7 @@ describe("ModalService", () => { await waitFor(() => userEvent.click(rendered.getByTestId("close-reason1"))); expect(rendered.queryByTestId("testModalContent")).toBeFalsy(); - expect(resultCallback).toHaveBeenCalledWith({ type: "closed", reason: "reason1" }); + expect(resultCallback).toHaveBeenCalledWith({ type: "completed", reason: "reason1" }); resultCallback.mockReset(); rendered = renderModal(resultCallback); @@ -88,6 +88,6 @@ describe("ModalService", () => { await waitFor(() => userEvent.click(rendered.getByTestId("close-reason2"))); expect(rendered.queryByTestId("testModalContent")).toBeFalsy(); - expect(resultCallback).toHaveBeenCalledWith({ type: "closed", reason: "reason2" }); + expect(resultCallback).toHaveBeenCalledWith({ type: "completed", reason: "reason2" }); }); }); diff --git a/airbyte-webapp/src/hooks/services/Modal/ModalService.tsx b/airbyte-webapp/src/hooks/services/Modal/ModalService.tsx index 48152bb4c2f..2f0d58b1439 100644 --- a/airbyte-webapp/src/hooks/services/Modal/ModalService.tsx +++ b/airbyte-webapp/src/hooks/services/Modal/ModalService.tsx @@ -19,18 +19,14 @@ export const ModalServiceProvider: React.FC> = const service: ModalServiceContext = useMemo( () => ({ - openModal: (options) => { + openModal: async (options) => { resultSubjectRef.current = new Subject(); setModalOptions(options); - return firstValueFrom(resultSubjectRef.current).then((reason) => { - setModalOptions(undefined); - resultSubjectRef.current = undefined; - return reason; - }); - }, - closeModal: () => { - resultSubjectRef.current?.next({ type: "canceled" }); + const reason = await firstValueFrom(resultSubjectRef.current); + setModalOptions(undefined); + resultSubjectRef.current = undefined; + return reason; }, }), [] @@ -45,11 +41,10 @@ export const ModalServiceProvider: React.FC> = size={modalOptions.size} testId={modalOptions.testId} onCancel={modalOptions.preventCancel ? undefined : () => resultSubjectRef.current?.next({ type: "canceled" })} - onClose={(reason) => resultSubjectRef.current?.next({ type: "closed", reason })} > resultSubjectRef.current?.next({ type: "canceled" })} - onClose={(reason) => resultSubjectRef.current?.next({ type: "closed", reason })} + onComplete={(result) => resultSubjectRef.current?.next({ type: "completed", reason: result })} />
)} diff --git a/airbyte-webapp/src/hooks/services/Modal/types.ts b/airbyte-webapp/src/hooks/services/Modal/types.ts index ffabcfcc789..48de57a3f01 100644 --- a/airbyte-webapp/src/hooks/services/Modal/types.ts +++ b/airbyte-webapp/src/hooks/services/Modal/types.ts @@ -10,14 +10,13 @@ export interface ModalOptions { testId?: string; } -export type ModalResult = { type: "canceled" } | { type: "closed"; reason: T }; +export type ModalResult = { type: "canceled" } | { type: "completed"; reason: T }; export interface ModalContentProps { - onClose: (reason: T) => void; + onComplete: (result: T) => void; onCancel: () => void; } export interface ModalServiceContext { openModal: (options: ModalOptions) => Promise>; - closeModal: () => void; } diff --git a/airbyte-webapp/src/hooks/useDeleteModal.tsx b/airbyte-webapp/src/hooks/useDeleteModal.tsx index a04926ab92c..c54e915c7a4 100644 --- a/airbyte-webapp/src/hooks/useDeleteModal.tsx +++ b/airbyte-webapp/src/hooks/useDeleteModal.tsx @@ -17,7 +17,12 @@ const routes: Routes = { connection: `../../../${RoutePaths.Connections}`, }; -export function useDeleteModal(entity: Entity, onDelete: () => Promise, additionalContent?: React.ReactNode) { +export function useDeleteModal( + entity: Entity, + onDelete: () => Promise, + additionalContent?: React.ReactNode, + confirmationText?: string +) { const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); const navigate = useNavigate(); @@ -26,6 +31,7 @@ export function useDeleteModal(entity: Entity, onDelete: () => Promise, text: `tables.${entity}DeleteModalText`, additionalContent, title: `tables.${entity}DeleteConfirm`, + confirmationText, submitButtonText: "form.delete", onSubmit: async () => { await onDelete(); @@ -34,5 +40,5 @@ export function useDeleteModal(entity: Entity, onDelete: () => Promise, }, submitButtonDataId: "delete", }); - }, [openConfirmationModal, entity, additionalContent, onDelete, closeConfirmationModal, navigate]); + }, [openConfirmationModal, entity, additionalContent, confirmationText, onDelete, closeConfirmationModal, navigate]); } diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index b4e2efc5e7a..c89263fed4b 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -153,10 +153,10 @@ "form.edit": "Edit", "form.done": "Done", "form.prefix": "Destination Stream Prefix", - "form.prefixNext": "Add a stream prefix", + "form.prefixNext": "Stream prefix", "form.prefix.message": "Add a prefix to stream names (ex. “airbyte_” causes “projects” => “airbyte_projects”)", "form.prefix.example": "example: ”projects” -> ”{prefix}projects”", - "form.prefix.subtitle": "Optional", + "form.prefix.subtitle": "Prefix text to each stream name in the destination", "form.prefix.placeholder": "prefix", "form.nameSearch": "Search stream name", "form.hideDisabledStreams": "Hide disabled streams", @@ -225,17 +225,18 @@ "connectionForm.nextButton": "Next", "connectionForm.configureConnection": "Configure connection", "connectionForm.selectStreams": "Select streams", + "connectionForm.selectStreams.readonly": "Selected streams", "connectionForm.selectSyncMode": "Select sync mode", "connectionForm.destinationNew": "Set up a new destination", "connectionForm.destinationNewDescription": "Configure a new destination from Airbyte's catalog of available connectors", "connectionForm.sourceFormat": "Mirror source structure", "connectionForm.sourceFormatNext": "Source-defined", "connectionForm.sourceFormatDescription": "Match the schema the source is in", - "connectionForm.sourceFormatDescriptionNext": "Use the schema(s) defined by the source.", + "connectionForm.sourceFormatDescriptionNext": "Use the schema(s) defined by the source.{sourceDefinedNamespaces}", "connectionForm.destinationFormat": "Destination default", "connectionForm.destinationFormatNext": "Destination-defined", "connectionForm.destinationFormatDescription": "Sync all streams to the default schema defined in the destination", - "connectionForm.destinationFormatDescriptionNext": "Sync all streams to the schema defined in the destination's settings.", + "connectionForm.destinationFormatDescriptionNext": "Sync all streams to {destinationDefinedNamespace, select, no_value_provided {} other {{destinationDefinedNamespace},}} the schema defined in the destination's settings.", "connectionForm.customFormat": "Custom format", "connectionForm.customFormatDescription": "Sync all streams to a unique new schema", "connectionForm.customFormatDescriptionNext": "Sync all streams to a unique new schema. Useful when syncing from multiple sources.", @@ -444,6 +445,11 @@ "jobs.jobStatus.reset_connection.succeeded": "Reset Succeeded ({count, plural, =0 {0 streams} one {# stream} other {# streams}})", "jobs.jobStatus.reset_connection.cancelled": "Reset Cancelled ({count, plural, =0 {0 streams} one {# stream} other {# streams}})", "jobs.jobStatus.reset_connection.partialSuccess": "Reset Partial Success ({count, plural, =0 {0 streams} one {# stream} other {# streams}})", + "jobs.jobStatus.clear_data.failed": "Clearing {count, plural, =0 {0 Streams} one {Stream} other {Streams}} Failed", + "jobs.jobStatus.clear_data.running": "Clearing {count, plural, =0 {0 Streams} one {Stream} other {Streams}} Running", + "jobs.jobStatus.clear_data.succeeded": "Clearing {count, plural, =0 {0 Streams} one {Stream} other {Streams}} Succeeded", + "jobs.jobStatus.clear_data.cancelled": "Clearing {count, plural, =0 {0 Streams} one {Stream} other {Streams}} Cancelled", + "jobs.jobStatus.clear_data.partialSuccess": "Clearing {count, plural, =0 {0 Streams} one {Stream} other {Streams}} Partially Succeeded", "jobs.jobStatus.sync.failed": "Sync Failed", "jobs.jobStatus.sync.running": "Sync Running", "jobs.jobStatus.sync.succeeded": "Sync Succeeded", @@ -605,6 +611,7 @@ "connection.pendingSync": "Sync is pending or running", "connection.refreshSchema": "Refresh schema", "connection.replication": "Replication", + "connection.schema": "Schema", "connection.streams": "Streams", "connection.transfer": "Transfer", "connection.linkCopied": "Link copied!", @@ -670,6 +677,14 @@ "connection.stream.status.table.emptyTable.message": "Re-enable the connection to show stream sync progress", "connection.stream.status.table.emptyTable.callToAction": "Re-enable", "connection.stream.actions.resetThisStream": "Reset this stream", + "connection.stream.actions.clearData": "Clear data", + "connection.stream.actions.clearData.confirm.title": "Are you sure you want to clear data from the {streamName} stream?", + "connection.actions.clearData.confirm.title": "Are you sure you want to clear data from this connection?", + "connection.actions.clearData.confirm.text": "Clearing data for this connection will delete all data in your destination for this connection.", + "connection.stream.actions.clearData.confirm.text": "Clearing data for this stream will delete all data in your destination for this stream.", + "connection.stream.actions.clearData.confirm.additionalText": "WARNING: This cannot be undone.", + "connection.stream.actions.clearData.confirm.submit": "Yes, clear data", + "connection.stream.actions.clearData.confirm.cancel": "No, cancel", "connection.stream.actions.showInReplicationTable": "Show in replication table", "connection.stream.actions.openDetails": "Open details", "connection.stream.status.nextSync": "Next sync {sync}", @@ -820,6 +835,8 @@ "settings.workspaceSettings": "Workspace", "settings.organizationSettings": "Organization", "settings.instanceSettings": "Instance", + "settings.workspace.general.title": "General workspace settings", + "settings.organization.general.title": "General organization settings", "settings.workspaceSettings.update.success": "Workspace settings have been updated!", "settings.workspaceSettings.update.error": "Something went wrong while updating your workspace settings. Please try again.", "settings.workspaceSettings.updateWorkspaceNameSuccess": "Workspace name has been updated!", @@ -881,7 +898,7 @@ "settings.notificationGuide.link.configuration": "Configure Sync notifications", "settings.notificationGuide.link.slackConfiguration": "Configure a Slack Notifications Webhook", "settings.metrics": "Metrics", - "settings.notificationSettings": "Notification Settings", + "settings.notificationSettings": "Notifications", "settings.metricsSettings": "Metrics Settings", "settings.emailNotifications": "Email notifications", "settings.securityUpdates": "Security updates (recommended)", @@ -891,11 +908,11 @@ "settings.cookiePreferences": "Cookie Preferences", "settings.dataResidency": "Data Residency", "settings.defaultDataResidency": "Default Data Residency", - "settings.geographyDescription": "Depending on your network configuration, you may need to add IP addresses to your allowlist. Request a new data residency.", "settings.defaultGeography": "Geography", - "settings.defaultDataResidencyDescription": "Choose the default preferred data processing location for all of your connections. The default data residency setting only affects new connections. Existing connections will retain their data residency setting. Learn more.", + "settings.defaultDataResidencyDescription": "Choose the default preferred data processing location for all of your connections. The default data residency setting only affects new connections. Existing connections will retain their data residency setting. Depending on your network configuration, you may need to add IP addresses to your allowlist. Request new data residency", "settings.defaultDataResidencyUpdateError": "There was an error updating the default data residency for this workspace.", "settings.defaultDataResidencyUpdateSuccess": "Data residency preference has been updated!", + "settings.general": "General", "settings.members": "Members", "settings.accessManagement.noUsers": "No users have this level of permission", "settings.accessManagement.removeUser": "Remove user", @@ -1010,6 +1027,10 @@ "connector.breakingChange.upgradeModal.moreConnections": "+ {count, plural, one {# more connection} other {# more connections}}", "connector.breakingChange.upgradeToast.success": "{type} upgraded successfully. See this guide for any remaining actions.", "connector.breakingChange.upgradeToast.failure": "Failed to upgrade {type}. Please reach out to support for assistance.", + "connector.check.failed": "Connection test were not successful.", + "connector.check.jobFailed": "Failed to run connection tests.", + "connector.discoverSchema.jobFailed": "Failed to run schema discovery.", + "connector.discoverSchema.catalogMissing": "Source did not return a schema.", "credits.credits": "Credits", "credits.whatAreCredits": "What are credits?", @@ -1095,7 +1116,7 @@ "connectorBuilder.recordsTab": "Records", "connectorBuilder.requestTab": "Request", "connectorBuilder.responseTab": "Response", - "connectorBuilder.schemaTab": "Detected schema", + "connectorBuilder.schemaTab": "Detected Schema", "connectorBuilder.useSchemaButton": "Import detected schema", "connectorBuilder.differentSchemaDescription": "Detected schema and declared schema are different", "connectorBuilder.overwriteSchemaButton": "Overwrite declared schema", @@ -1186,7 +1207,7 @@ "connectorBuilder.duplicateFieldID": "Make sure no field ID is used multiple times", "connectorBuilder.addNewParentStream": "Add new parent stream", "connectorBuilder.streamConfiguration": "Configuration", - "connectorBuilder.streamSchema": "Declared schema", + "connectorBuilder.streamSchema": "Declared Schema", "connectorBuilder.invalidJSON": "Invalid JSON - please fix syntax to have it applied", "connectorBuilder.copyToPaginationTitle": "Copy pagination settings to...", "connectorBuilder.copyFromPaginationTitle": "Import pagination settings from...", @@ -1441,6 +1462,11 @@ "connectorBuilder.transformation.remove.path.tooltip": "Path to the field to remove", "connectorBuilder.transformation.add": "Add Field", "connectorBuilder.adminTestingValuesWarning": "Admin: changes to testing values will be saved to the database, which the user will see if they return to this project.", + "connectorBuilder.yamlComponent.discardChanges.title": "Discard YAML Changes?", + "connectorBuilder.yamlComponent.discardChanges.unknownErrorIntro": "The YAML you entered is not supported in UI form.", + "connectorBuilder.yamlComponent.discardChanges.knownErrorIntro": "The YAML you entered is not supported in UI form due to the following error:", + "connectorBuilder.yamlComponent.discardChanges.errorOutro": "Click confirm to discard your YAML changes and revert back to the previous UI state.", + "connectorBuilder.yamlComponent.discardChanges.confirm": "Confirm", "jobs.noAttemptsFailure": "Failed to start job.", @@ -1479,6 +1505,7 @@ "auth.authError": "An error occurred during authentication: {errorMessage}", "modal.closeButtonLabel": "Close dialog", + "modal.confirmationTextDescription": "Type {confirmationText} below to confirm this action.", "copyButton.title": "Copy", @@ -1489,6 +1516,8 @@ "login.returnToLogin": "Return to login", "login.signup.submitButton": "Sign up", "login.loginTitle": "Log in to Airbyte", + "login.acceptInvite": "You've been invited to collaborate on Airbyte", + "login.acceptInvite.subtitle": "Log in or sign up to get started", "login.resendEmail": "Didn't receive the email? Send it again", "login.yourEmail": "Your work email*", "login.inviteEmail": "For security, re-enter your invite email*", @@ -1653,7 +1682,8 @@ "credits.creditsProblem": "You’re out of credits! To set up connections and run syncs, add credits.", "credits.emailVerificationRequired": "You need to verify your email address before you can buy credits.", "credits.emailVerification.resendConfirmation": "We sent you a new verification link.", - "credits.emailVerification.resend": "Send verification link again", + "credits.emailVerification.resendConfirmationError": "There was an error sending the verification link. Please try again.", + "credits.emailVerification.resend": "Send verification link", "credits.lowBalance": "Your credit balance is low. Buy more credits to prevent your connections from being disabled or enroll in auto-recharge.", "credits.zeroBalance": "All your connections have been disabled because your credit balance is 0. Buy credits or enroll in auto-recharge to enable your data to sync.", @@ -1683,12 +1713,16 @@ "userInvitations.accept.success": "Invitation accepted successfully", "userInvitations.accept.error": "Something went wrong accepting the invitation. Please check your link.", + "userInvitations.accept.error.email": "Current user's email address does not match invitation.", + "userInvitations.accept.error.expired": "This invitation has expired. Please request a new one.", + "userInvitations.accept.error.cancelled": "This invitation has been cancelled.", + "userInvitations.accept.warning.alreadyAccepted": "You have already accepted this invitation.", "userInvitations.create.modal.title": "Add a member to {workspace}", "userInvitations.create.modal.addNew": "Add new member", - "userInvitations.create.modal.existingMember": "This member already exists. Go to workspace settings to edit this user’s permissions.", "userInvitations.create.modal.search": "Type to add a new member", "userInvitations.create.success": "Invitation created successfully", "userInvitations.create.error": "There was an error inviting this user. Please try again.", + "userInvitations.create.error.duplicate": "There is already a pending invitation for this email.", "userInvitations.create.modal.emptyList": "No matching users found", "userInvitations.create.modal.emptyList.canInvite": "No matching users found. Enter a valid email address to invite a new member.", "userInvitations.create.modal.emptyList.noOrganization": "Enter a valid email address to invite a new member.", @@ -1697,5 +1731,10 @@ "userInvitations.create.modal.organizationAdminTooltip": "This user already has full access to this workspace.", "userInvitations.pendingInvitation": "Pending...", "userInvitations.pendingInvitation.tooltipMain": "This member has not yet accepted their invitation.", - "userInvitations.pendingInvitation.tooltipAdditionalInfo": "They do not yet have access to this workspace." + "userInvitations.pendingInvitation.tooltipAdditionalInfo": "They do not yet have access to this workspace.", + "userInvitations.create.modal.asRole": "As {role}", + "userInvitations.cancel.success": "User invitation successfully cancelled", + "userInvitations.cancel.error": "There was an error cancelling this invitation. Please try again.", + "userInvitations.cancel.confirm.text": "Are you sure you want to cancel this invitation for {user} to {resource}?", + "userInvitations.cancel.confirm.title": "Cancel invitation" } diff --git a/airbyte-webapp/src/packages/cloud/App.tsx b/airbyte-webapp/src/packages/cloud/App.tsx index 85e9c727c13..6bbbdbc38fb 100644 --- a/airbyte-webapp/src/packages/cloud/App.tsx +++ b/airbyte-webapp/src/packages/cloud/App.tsx @@ -8,7 +8,6 @@ import { DevToolsToggle } from "components/DevToolsToggle"; import LoadingPage from "components/LoadingPage"; import { QueryProvider } from "core/api"; -import { ConfigServiceProvider, config } from "core/config"; import { AnalyticsProvider } from "core/services/analytics"; import { defaultCloudFeatures, FeatureService } from "core/services/features"; import { I18nProvider } from "core/services/i18n"; @@ -52,18 +51,16 @@ const App: React.FC = () => { }> - - - - - - - - - - - - + + + + + + + + + + diff --git a/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx b/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx index 83645f637bf..1e6cf10551b 100644 --- a/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx +++ b/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx @@ -181,7 +181,7 @@ export const Routing: React.FC = () => { })}`; const loginRedirectTo = - loggedOut && (originalPathname === "/" || originalPathname.includes("/settings/account")) + loggedOut && originalPathname === "/" ? { pathname: CloudRoutes.Login } : { pathname: CloudRoutes.Login, search: loginRedirectSearchParam }; diff --git a/airbyte-webapp/src/packages/cloud/services/FirebaseSdkProvider.tsx b/airbyte-webapp/src/packages/cloud/services/FirebaseSdkProvider.tsx index 6d3c54fa58c..6ab6e3a1467 100644 --- a/airbyte-webapp/src/packages/cloud/services/FirebaseSdkProvider.tsx +++ b/airbyte-webapp/src/packages/cloud/services/FirebaseSdkProvider.tsx @@ -1,11 +1,10 @@ import { getAuth, connectAuthEmulator } from "firebase/auth"; import React from "react"; -import { useConfig } from "core/config"; +import { config } from "core/config"; import { FirebaseAppProvider, useFirebaseApp, AuthProvider } from "packages/firebaseReact"; const FirebaseAppSdksProvider: React.FC> = ({ children }) => { - const config = useConfig(); const firebaseApp = useFirebaseApp(); const auth = getAuth(firebaseApp); if (config.firebase.authEmulatorHost) { @@ -20,8 +19,6 @@ const FirebaseAppSdksProvider: React.FC> = ({ c * based on airbyte app config and also injecting all required firebase sdks */ const FirebaseSdkProvider: React.FC> = ({ children }) => { - const config = useConfig(); - return ( {children} diff --git a/airbyte-webapp/src/packages/cloud/services/auth/CloudAuthService.tsx b/airbyte-webapp/src/packages/cloud/services/auth/CloudAuthService.tsx index 4697ea688c8..a917af07521 100644 --- a/airbyte-webapp/src/packages/cloud/services/auth/CloudAuthService.tsx +++ b/airbyte-webapp/src/packages/cloud/services/auth/CloudAuthService.tsx @@ -186,38 +186,7 @@ export const CloudAuthService: React.FC = ({ children }) => { console.error("sendEmailVerifiedLink should be used within auth flow"); throw new Error("Cannot send verification email if firebaseUser is null."); } - return sendEmailVerification(firebaseUser) - .then(() => { - registerNotification({ - id: "workspace.emailVerificationResendSuccess", - text: , - type: "success", - }); - }) - .catch((error) => { - switch (error.code) { - case AuthErrorCodes.NETWORK_REQUEST_FAILED: - registerNotification({ - id: error.code, - text: , - type: "error", - }); - break; - case AuthErrorCodes.TOO_MANY_ATTEMPTS_TRY_LATER: - registerNotification({ - id: error.code, - text: , - type: "warning", - }); - break; - default: - registerNotification({ - id: error.code, - text: , - type: "error", - }); - } - }); + return sendEmailVerification(firebaseUser); }, verifyEmail: verifyFirebaseEmail, }; @@ -389,7 +358,6 @@ export const CloudAuthService: React.FC = ({ children }) => { getAirbyteUser, keycloakAuth, logout, - registerNotification, resendWithSignInLink, updateAirbyteUser, verifyFirebaseEmail, diff --git a/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/LDExperimentService.tsx b/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/LDExperimentService.tsx index c54fe935655..7a2ac7cf94c 100644 --- a/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/LDExperimentService.tsx +++ b/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/LDExperimentService.tsx @@ -7,7 +7,7 @@ import { finalize, Subject } from "rxjs"; import { LoadingPage } from "components"; import { useCurrentWorkspaceId } from "area/workspace/utils"; -import { useConfig } from "core/config"; +import { config } from "core/config"; import { useAnalyticsService } from "core/services/analytics"; import { useAuthService } from "core/services/auth"; import { FeatureSet, FeatureItem, useFeatureService } from "core/services/features"; @@ -234,7 +234,7 @@ const LDInitializationWrapper: React.FC> = ({ children }) => { - const { launchDarkly: launchdarklyKey } = useConfig(); + const { launchDarkly: launchdarklyKey } = config; return !launchdarklyKey ? ( <>{children} diff --git a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx index 4a2bd075f22..f1a9a694a3c 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx @@ -9,7 +9,7 @@ import { HeadTitle } from "components/common/HeadTitle"; import { Form, FormControl } from "components/forms"; import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; -import { FlexContainer } from "components/ui/Flex"; +import { FlexContainer, FlexItem } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { Link } from "components/ui/Link"; import { Text } from "components/ui/Text"; @@ -56,6 +56,7 @@ export const LoginPage: React.FC = () => { const [searchParams] = useSearchParams(); const [keycloakAuthEnabled] = useLocalStorage("airbyte_keycloak-auth-ui", false); const loginRedirectString = searchParams.get("loginRedirect"); + const isAcceptingInvitation = loginRedirectString?.includes("accept-invite"); const navigate = useNavigate(); @@ -98,9 +99,18 @@ export const LoginPage: React.FC = () => { return ( - - - + + + + + {isAcceptingInvitation && ( + + + + + + )} + {loginWithOAuth && ( <> diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/BillingBanners.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/BillingBanners.tsx index 9bdf2b57a6c..64dca5234c9 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/BillingBanners.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/BillingBanners.tsx @@ -5,29 +5,24 @@ import { ExternalLink, Link } from "components/ui/Link"; import { Message } from "components/ui/Message"; import { useCurrentWorkspace } from "core/api"; -import { useGetCloudWorkspace } from "core/api/cloud"; +import { useGetCloudWorkspace, useResendEmailVerification } from "core/api/cloud"; import { CloudWorkspaceReadCreditStatus, CloudWorkspaceReadWorkspaceTrialStatus } from "core/api/types/CloudApi"; -import { AuthSendEmailVerification, useAuthService } from "core/services/auth"; +import { useAuthService } from "core/services/auth"; import { links } from "core/utils/links"; import { useExperiment } from "hooks/services/Experiment"; const LOW_BALANCE_CREDIT_THRESHOLD = 20; -interface EmailVerificationHintProps { - sendEmailVerification: AuthSendEmailVerification; -} - -export const EmailVerificationHint: React.FC = ({ sendEmailVerification }) => { - const onResendVerificationMail = async () => { - return sendEmailVerification(); - }; +export const EmailVerificationHint: React.FC = () => { + const { mutateAsync: resendEmailVerification, isLoading } = useResendEmailVerification(); return ( } actionBtnText={} - onAction={onResendVerificationMail} + actionBtnProps={{ isLoading }} + onAction={resendEmailVerification} /> ); }; @@ -99,15 +94,13 @@ const LowCreditBalanceHint: React.FC = () => { }; export const BillingBanners: React.FC = () => { - const { sendEmailVerification, emailVerified } = useAuthService(); + const { emailVerified } = useAuthService(); const isAutoRechargeEnabled = useExperiment("billing.autoRecharge", false); return ( - {!emailVerified && sendEmailVerification && ( - - )} + {!emailVerified && } {isAutoRechargeEnabled ? : } ); diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx index f49b04b2207..a946b6a5ce4 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx @@ -1,9 +1,9 @@ import dayjs from "dayjs"; -import { Dispatch, SetStateAction, createContext, useContext, useMemo, useState } from "react"; +import { createContext, useContext, useMemo, useState } from "react"; import { Option } from "components/ui/ListBox"; -import { useCurrentWorkspace } from "core/api"; +import { useCurrentWorkspace, useFilters } from "core/api"; import { useGetCloudWorkspaceUsage } from "core/api/cloud"; import { DestinationId, SourceId, SupportLevel } from "core/api/types/AirbyteClient"; import { ConsumptionTimeWindow } from "core/api/types/CloudApi"; @@ -42,10 +42,10 @@ interface CreditsUsageContext { destinationOptions: Array>; selectedSource: SourceId | null; selectedDestination: DestinationId | null; - setSelectedSource: Dispatch>; - setSelectedDestination: Dispatch>; + setSelectedSource: (sourceId: SourceId | null) => void; + setSelectedDestination: (destinationId: DestinationId | null) => void; selectedTimeWindow: ConsumptionTimeWindow; - setSelectedTimeWindow: Dispatch>; + setSelectedTimeWindow: (timeWindow: ConsumptionTimeWindow) => void; hasFreeUsage: boolean; } @@ -59,12 +59,24 @@ export const useCreditsContext = (): CreditsUsageContext => { return creditsUsageHelpers; }; +interface FilterValues { + selectedTimeWindow: ConsumptionTimeWindow; + selectedSource: SourceId | null; + selectedDestination: DestinationId | null; +} + export const CreditsUsageContextProvider: React.FC> = ({ children }) => { - const [selectedTimeWindow, setSelectedTimeWindow] = useState(ConsumptionTimeWindow.lastMonth); + const [filters, setFilterValue] = useFilters({ + selectedTimeWindow: ConsumptionTimeWindow.lastMonth, + selectedSource: null, + selectedDestination: null, + }); + const { selectedTimeWindow, selectedSource, selectedDestination } = filters; + const [hasFreeUsage, setHasFreeUsage] = useState(false); const { workspaceId } = useCurrentWorkspace(); - const data = useGetCloudWorkspaceUsage(workspaceId, selectedTimeWindow); + const data = useGetCloudWorkspaceUsage(workspaceId, filters.selectedTimeWindow); const { consumptionPerConnectionPerTimeframe, timeWindow } = data; @@ -82,8 +94,6 @@ export const CreditsUsageContextProvider: React.FC(null); - const [selectedDestination, setSelectedDestination] = useState(null); const availableSourcesAndDestinations = useMemo( () => calculateAvailableSourcesAndDestinations(rawConsumptionData), [rawConsumptionData] @@ -135,11 +145,13 @@ export const CreditsUsageContextProvider: React.FC setFilterValue("selectedSource", selectedSource), selectedDestination, - setSelectedDestination, + setSelectedDestination: (selectedDestination: DestinationId | null) => + setFilterValue("selectedDestination", selectedDestination), selectedTimeWindow, - setSelectedTimeWindow, + setSelectedTimeWindow: (selectedTimeWindow: ConsumptionTimeWindow) => + setFilterValue("selectedTimeWindow", selectedTimeWindow), hasFreeUsage, }} > diff --git a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx index d698bf6d4b0..f174d47b853 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx @@ -14,7 +14,6 @@ import { SideBar } from "views/layout/SideBar/SideBar"; import { CloudHelpDropdown } from "./CloudHelpDropdown"; import styles from "./CloudMainView.module.scss"; -import { InsufficientPermissionsErrorBoundary } from "./InsufficientPermissionsErrorBoundary"; import { WorkspaceStatusBanner } from "./WorkspaceStatusBanner"; const CloudMainView: React.FC = (props) => { @@ -25,17 +24,15 @@ const CloudMainView: React.FC = (props) => { return ( - } trackError={trackError}> - {cloudWorkspace && } - - } /> -
- } trackError={trackError}> - }>{props.children ?? } - -
- - + {cloudWorkspace && } + + } /> +
+ } trackError={trackError}> + }>{props.children ?? } + +
+ ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/InsufficientPermissionsErrorBoundary.tsx b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/InsufficientPermissionsErrorBoundary.tsx deleted file mode 100644 index b81c25c1577..00000000000 --- a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/InsufficientPermissionsErrorBoundary.tsx +++ /dev/null @@ -1,50 +0,0 @@ -import React from "react"; - -import { CommonRequestError } from "core/api"; -import { TrackErrorFn } from "hooks/services/AppMonitoringService"; - -interface BoundaryState { - hasError: boolean; - message?: React.ReactNode | null; -} - -const initialState: BoundaryState = { - hasError: false, - message: null, -}; - -interface InsufficientPermissionsErrorBoundaryProps { - errorComponent: React.ReactElement; - trackError: TrackErrorFn; -} - -export class InsufficientPermissionsErrorBoundary extends React.Component< - React.PropsWithChildren, - BoundaryState -> { - static getDerivedStateFromError(error: CommonRequestError): BoundaryState { - if (error.message.startsWith("Insufficient permissions")) { - return { hasError: true, message: error.message }; - } - throw error; - } - - componentDidCatch(error: Error): void { - this.props.trackError(error, { errorBoundary: this.constructor.name }); - } - - state = initialState; - - reset = (): void => { - this.setState(initialState); - }; - - render(): React.ReactNode { - return this.state.hasError - ? React.cloneElement(this.props.errorComponent, { - message: this.state.message, - onReset: this.reset, - }) - : this.props.children; - } -} diff --git a/airbyte-webapp/src/packages/cloud/views/settings/CloudSettingsPage.tsx b/airbyte-webapp/src/packages/cloud/views/settings/CloudSettingsPage.tsx index 3322ade2616..a0f20c82f8c 100644 --- a/airbyte-webapp/src/packages/cloud/views/settings/CloudSettingsPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/settings/CloudSettingsPage.tsx @@ -1,19 +1,16 @@ import React, { Suspense } from "react"; -import { FormattedMessage, useIntl } from "react-intl"; +import { useIntl } from "react-intl"; import { Outlet } from "react-router-dom"; -import { LoadingPage, MainPageWithScroll } from "components"; -import { HeadTitle } from "components/common/HeadTitle"; +import { LoadingPage } from "components"; + +import { SettingsLayout, SettingsLayoutContent } from "area/settings/components/SettingsLayout"; import { SettingsButton, SettingsLink, SettingsNavigation, SettingsNavigationBlock, -} from "components/settings/SettingsNavigation"; -import { FlexContainer, FlexItem } from "components/ui/Flex"; -import { Heading } from "components/ui/Heading"; -import { PageHeader } from "components/ui/PageHeader"; - +} from "area/settings/components/SettingsNavigation"; import { useCurrentOrganizationInfo } from "core/api"; import { FeatureItem, useFeature } from "core/services/features"; import { isOsanoActive, showOsanoDrawer } from "core/utils/dataPrivacy"; @@ -32,101 +29,88 @@ export const CloudSettingsPage: React.FC = () => { const showAdvancedSettings = useExperiment("settings.showAdvancedSettings", false); return ( - } - pageTitle={ - - - - } - /> - } - > - - - + + + + + {isTokenManagementEnabled && ( - {isTokenManagementEnabled && ( - - )} - {isOsanoActive() && ( - showOsanoDrawer()} - name={formatMessage({ id: "settings.cookiePreferences" })} - /> - )} - {showAdvancedSettings && ( - - )} - - + )} + {isOsanoActive() && ( + showOsanoDrawer()} + name={formatMessage({ id: "settings.cookiePreferences" })} + /> + )} + {showAdvancedSettings && ( - {supportsDataResidency && ( - - )} + )} + + + + {supportsDataResidency && ( + )} + + + {supportsCloudDbtIntegration && ( - {supportsCloudDbtIntegration && ( - - )} + )} + + + {organization && canViewOrgSettings && ( + - {organization && canViewOrgSettings && ( - - - - )} - - - }> - - - - - + )} + + + }> + + + + ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/settings/integrations/DbtCloudSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/settings/integrations/DbtCloudSettingsView.tsx index 0ebaa5bc011..891268aa45e 100644 --- a/airbyte-webapp/src/packages/cloud/views/settings/integrations/DbtCloudSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/settings/integrations/DbtCloudSettingsView.tsx @@ -5,8 +5,8 @@ import * as yup from "yup"; import { Form, FormControl } from "components/forms"; import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; import { Button } from "components/ui/Button"; -import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; import { ExternalLink } from "components/ui/Link"; import { Text } from "components/ui/Text"; @@ -61,47 +61,46 @@ export const DbtCloudSettingsView: React.FC = () => { }; return ( - - - - {node}, - }} - /> - - - defaultValues={{ serviceToken: "" }} - onSubmit={onSubmit} - onSuccess={onSuccess} - onError={onError} - schema={ServiceTokenFormSchema} - disabled={!canUpdateWorkspace} - > - - {hasExistingToken ? ( - - - - ) : ( - - )} - - - + + {formatMessage({ id: "settings.integrationSettings.dbtCloudSettings" })} + + {node}, + }} + /> + + + defaultValues={{ serviceToken: "" }} + onSubmit={onSubmit} + onSuccess={onSuccess} + onError={onError} + schema={ServiceTokenFormSchema} + disabled={!canUpdateWorkspace} + > + + {hasExistingToken ? ( + + + + ) : ( + + )} + + ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx index cae4614a1b1..a28c88bac48 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx @@ -1,24 +1,28 @@ import React from "react"; import { FlexContainer } from "components/ui/Flex"; +import { Separator } from "components/ui/Separator"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { useAuthService } from "core/services/auth"; import { EmailSection, NameSection, PasswordSection } from "./components"; -import { LogoutSection } from "./components/LogoutSection"; export const AccountSettingsView: React.FC = () => { - const { logout, updateName, hasPasswordLogin, updatePassword } = useAuthService(); + const { updateName, hasPasswordLogin, updatePassword } = useAuthService(); useTrackPage(PageTrackingCodes.SETTINGS_ACCOUNT); return ( - - {updateName && } + - {hasPasswordLogin?.() && updatePassword && } - {logout && } + {updateName && } + {hasPasswordLogin?.() && updatePassword && ( + <> + + + + )} ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.module.scss b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.module.scss new file mode 100644 index 00000000000..4e4d43fd64e --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.module.scss @@ -0,0 +1,3 @@ +.emailControl { + padding-bottom: 0; +} \ No newline at end of file diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.tsx index 409d10ff94c..c3dd33a3e7c 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.tsx @@ -3,10 +3,11 @@ import { useIntl } from "react-intl"; import * as yup from "yup"; import { Form, FormControl } from "components/forms"; -import { Card } from "components/ui/Card"; import { useCurrentUser } from "core/services/auth"; +import styles from "./EmailSection.module.scss"; + const emailFormSchema = yup.object({ email: yup.string().required("form.empty.error"), }); @@ -20,27 +21,26 @@ export const EmailSection: React.FC = () => { const user = useCurrentUser(); return ( - - - defaultValues={{ - email: user.email, - }} - schema={emailFormSchema} - > - - name="email" - fieldType="input" - type="text" - label={formatMessage({ id: "settings.accountSettings.email" })} - placeholder={formatMessage({ - id: "login.yourEmail.placeholder", - })} - /* + + defaultValues={{ + email: user.email, + }} + schema={emailFormSchema} + > + + containerControlClassName={styles.emailControl} + name="email" + fieldType="input" + type="text" + label={formatMessage({ id: "settings.accountSettings.email" })} + placeholder={formatMessage({ + id: "login.yourEmail.placeholder", + })} + /* show user's email in read-only mode, details: https://github.com/airbytehq/airbyte-platform-internal/issues/1269 */ - disabled - /> - - + disabled + /> + ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/LogoutSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/LogoutSection.tsx deleted file mode 100644 index a8ede60ab20..00000000000 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/LogoutSection.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import { useMutation } from "@tanstack/react-query"; -import { FormattedMessage } from "react-intl"; - -import { Box } from "components/ui/Box"; -import { Button } from "components/ui/Button"; -import { FlexContainer } from "components/ui/Flex"; - -import { AuthLogout } from "core/services/auth"; - -export const LogoutSection = ({ logout }: { logout: AuthLogout }) => { - const { mutateAsync: doLogout, isLoading: isLoggingOut } = useMutation(() => logout()); - - return ( - - - - - - ); -}; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection.tsx index 1ad9d220726..1deaef54d69 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection.tsx @@ -4,7 +4,6 @@ import * as yup from "yup"; import { Form, FormControl } from "components/forms"; import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; -import { Card } from "components/ui/Card"; import { AuthChangeName, useCurrentUser } from "core/services/auth"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; @@ -46,24 +45,22 @@ export const NameSection: React.FC = ({ updateName }) => { }; return ( - - - onSubmit={({ name }) => updateName(name)} - onError={onError} - onSuccess={onSuccess} - schema={nameFormSchema} - defaultValues={{ name: user.name }} - > - - label={formatMessage({ id: "settings.accountSettings.name" })} - fieldType="input" - name="name" - placeholder={formatMessage({ - id: "settings.accountSettings.name.placeholder", - })} - /> - - - + + onSubmit={({ name }) => updateName(name)} + onError={onError} + onSuccess={onSuccess} + schema={nameFormSchema} + defaultValues={{ name: user.name }} + > + + label={formatMessage({ id: "settings.accountSettings.name" })} + fieldType="input" + name="name" + placeholder={formatMessage({ + id: "settings.accountSettings.name.placeholder", + })} + /> + + ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection.tsx index 46b34c08be1..37c60b2fb6a 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection.tsx @@ -5,7 +5,6 @@ import * as yup from "yup"; import { Form, FormControl } from "components/forms"; import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; -import { Card } from "components/ui/Card"; import { AuthUpdatePassword, useCurrentUser } from "core/services/auth"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; @@ -98,39 +97,37 @@ export const PasswordSection: React.FC = ({ updatePassword }; return ( - - - defaultValues={defaultFormValues} - onSubmit={onSubmit} - onSuccess={onSuccess} - onError={onError} - schema={passwordFormSchema} - > - - label={formatMessage({ id: "settings.accountSettings.currentPassword" })} - name="currentPassword" - type="password" - fieldType="input" - required - autoComplete="current-password" - /> - - label={formatMessage({ id: "settings.accountSettings.newPassword" })} - name="newPassword" - type="password" - fieldType="input" - required - autoComplete="new-password" - /> - - label={formatMessage({ id: "settings.accountSettings.newPasswordConfirmation" })} - name="passwordConfirmation" - type="password" - fieldType="input" - required - /> - - - + + defaultValues={defaultFormValues} + onSubmit={onSubmit} + onSuccess={onSuccess} + onError={onError} + schema={passwordFormSchema} + > + + label={formatMessage({ id: "settings.accountSettings.currentPassword" })} + name="currentPassword" + type="password" + fieldType="input" + required + autoComplete="current-password" + /> + + label={formatMessage({ id: "settings.accountSettings.newPassword" })} + name="newPassword" + type="password" + fieldType="input" + required + autoComplete="new-password" + /> + + label={formatMessage({ id: "settings.accountSettings.newPasswordConfirmation" })} + name="passwordConfirmation" + type="password" + fieldType="input" + required + /> + + ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/users/ApplicationSettingsView/ApplicationSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/ApplicationSettingsView/ApplicationSettingsView.tsx index 62f7ad6bc81..69e5c5536f3 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/ApplicationSettingsView/ApplicationSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/ApplicationSettingsView/ApplicationSettingsView.tsx @@ -4,7 +4,6 @@ import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; -import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { Table } from "components/ui/Table"; @@ -85,10 +84,10 @@ export const ApplicationSettingsView = () => { }, [columnHelper]); return ( - + <> - + @@ -101,7 +100,7 @@ export const ApplicationSettingsView = () => { {applications.length ? ( - +
) : ( @@ -110,6 +109,6 @@ export const ApplicationSettingsView = () => { )} - + ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/users/ApplicationSettingsView/CreateApplicationControl.tsx b/airbyte-webapp/src/packages/cloud/views/users/ApplicationSettingsView/CreateApplicationControl.tsx index 13348fc02d6..05d3e6ba029 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/ApplicationSettingsView/CreateApplicationControl.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/ApplicationSettingsView/CreateApplicationControl.tsx @@ -16,22 +16,24 @@ export const CreateApplicationControl = () => { const { formatMessage } = useIntl(); const { mutateAsync: createApplication } = useCreateApplication(); const { applications } = useListApplications(); - const { openModal, closeModal } = useModalService(); + const { openModal } = useModalService(); const schema = yup.object().shape({ name: yup.string().required("form.empty.error"), }); - const onCreateApplicationSubmission = async (values: ApplicationCreate) => { - await createApplication(values); - closeModal(); - }; - - const onAddApplicationButtonClick = async () => { - openModal({ + const onAddApplicationButtonClick = () => + openModal({ title: formatMessage({ id: "settings.application.create" }), - content: () => ( - schema={schema} defaultValues={{ name: "" }} onSubmit={onCreateApplicationSubmission}> + content: ({ onComplete, onCancel }) => ( + + schema={schema} + defaultValues={{ name: "" }} + onSubmit={async (values: ApplicationCreate) => { + await createApplication(values); + onComplete(); + }} + > { /> - + ), size: "md", }); - }; return ( <> diff --git a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersHint/InviteUsersHint.tsx b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersHint/InviteUsersHint.tsx index da814435061..46f28dd40f1 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersHint/InviteUsersHint.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersHint/InviteUsersHint.tsx @@ -29,9 +29,11 @@ export const InviteUsersHint: React.FC = ({ connectorType } const onOpenInviteUsersModal = () => - openModal({ + openModal({ title: formatMessage({ id: "modals.addUser.title" }), - content: () => , + content: ({ onComplete, onCancel }) => ( + + ), size: "md", }); diff --git a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx index ba65d36f16d..9801822bdb6 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx @@ -11,7 +11,6 @@ import { ModalBody, ModalFooter } from "components/ui/Modal"; import { useUserHook } from "core/api/cloud"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { trackError } from "core/utils/datadog"; -import { useModalService } from "hooks/services/Modal"; import { useNotificationService } from "hooks/services/Notification"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; @@ -35,17 +34,18 @@ const requestConnectorValidationSchema: SchemaOf = yup.ob export const InviteUsersModal: React.FC<{ invitedFrom: "source" | "destination" | "user.settings"; -}> = ({ invitedFrom }) => { + onSubmit: () => void; + onCancel: () => void; +}> = ({ invitedFrom, onSubmit, onCancel }) => { const { formatMessage } = useIntl(); const { workspaceId } = useCurrentWorkspace(); const { inviteUserLogic } = useUserHook(); const { mutateAsync: invite } = inviteUserLogic; - const { closeModal } = useModalService(); const { registerNotification } = useNotificationService(); const analyticsService = useAnalyticsService(); - const onSubmit = async (values: InviteUsersFormValues) => { + const onSubmitBtnClick = async (values: InviteUsersFormValues) => { await invite({ users: values.users, workspaceId }); analyticsService.track(Namespace.USER, Action.INVITE, { @@ -59,7 +59,7 @@ export const InviteUsersModal: React.FC<{ text: formatMessage({ id: "inviteUsers.invitationsSentSuccess" }), type: "success", }); - closeModal(); + onSubmit(); }; const onError = (e: Error, { users }: InviteUsersFormValues) => { @@ -84,7 +84,7 @@ export const InviteUsersModal: React.FC<{ schema={requestConnectorValidationSchema} defaultValues={formDefaultValues} - onSubmit={onSubmit} + onSubmit={onSubmitBtnClick} onSuccess={onSuccess} onError={onError} > @@ -94,7 +94,7 @@ export const InviteUsersModal: React.FC<{ - + ); diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/DataResidencyView/DataResidencyView.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/DataResidencyView/DataResidencyView.tsx index be6cb52943a..49a0f1963ac 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/DataResidencyView/DataResidencyView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/DataResidencyView/DataResidencyView.tsx @@ -5,8 +5,8 @@ import * as yup from "yup"; import { Form } from "components/forms"; import { DataResidencyDropdown } from "components/forms/DataResidencyDropdown"; import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; -import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; import { ExternalLink } from "components/ui/Link"; import { Text } from "components/ui/Text"; @@ -27,16 +27,6 @@ interface DefaultDataResidencyFormValues { defaultGeography?: Geography; } -const fieldDescription = ( - {node}, - request: (node: React.ReactNode) => {node}, - }} - /> -); - export const DataResidencyView: React.FC = () => { const workspace = useCurrentWorkspace(); const { mutateAsync: updateWorkspace } = useUpdateWorkspace(); @@ -70,35 +60,33 @@ export const DataResidencyView: React.FC = () => { }; return ( - - - - {node}, - }} - /> - - - defaultValues={{ - defaultGeography: workspace.defaultGeography, + + {formatMessage({ id: "settings.defaultDataResidency" })} + + {node}, + request: (node: React.ReactNode) => {node}, }} - schema={schema} - onSubmit={handleSubmit} - onSuccess={onSuccess} - onError={onError} - disabled={!canUpdateWorkspace} - > - - labelId="settings.defaultGeography" - description={fieldDescription} - name="defaultGeography" - inline - /> - - - - + /> + + + defaultValues={{ + defaultGeography: workspace.defaultGeography, + }} + schema={schema} + onSubmit={handleSubmit} + onSuccess={onSuccess} + onError={onError} + disabled={!canUpdateWorkspace} + > + + labelId="settings.defaultGeography" + name="defaultGeography" + /> + + + ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx index ff55d4938d8..a8662cfa5d2 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx @@ -1,10 +1,9 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import { Box } from "components/ui/Box"; -import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; +import { Separator } from "components/ui/Separator"; import { useCurrentWorkspace } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; @@ -22,32 +21,31 @@ export const WorkspaceSettingsView: React.FC = () => { const isAccessManagementEnabled = useFeature(FeatureItem.RBAC); return ( - - - - - - - - + + + + + {isAccessManagementEnabled && ( - + <> + - + )} {canDeleteWorkspace && ( - + <> + - + - + )} ); diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModal.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModal.tsx index 58fc9c5e6d5..30391631bba 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModal.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModal.tsx @@ -1,8 +1,8 @@ import { useDeferredValue, useMemo, useState } from "react"; import { useFormState } from "react-hook-form"; import { FormattedMessage, useIntl } from "react-intl"; -import { SchemaOf } from "yup"; import * as yup from "yup"; +import { SchemaOf } from "yup"; import { Form } from "components/forms"; import { Box } from "components/ui/Box"; @@ -19,6 +19,7 @@ import { } from "core/api"; import { PermissionType, WorkspaceUserAccessInfoRead } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; +import { useIntent } from "core/utils/rbac"; import { AddUserModalBody } from "./AddUserModalBody"; @@ -42,11 +43,16 @@ const SubmissionButton: React.FC = () => { ); }; -export const AddUserModal: React.FC<{ closeModal: () => void }> = ({ closeModal }) => { +export const AddUserModal: React.FC<{ onSubmit: () => void }> = ({ onSubmit }) => { const { formatMessage } = useIntl(); const workspaceId = useCurrentWorkspaceId(); const organizationInfo = useCurrentOrganizationInfo(); - const { users } = useListUsersInOrganization(organizationInfo?.organizationId); + const canListUsersInOrganization = useIntent("ListOrganizationMembers", { + organizationId: organizationInfo?.organizationId, + }); + const { users } = useListUsersInOrganization( + canListUsersInOrganization ? organizationInfo?.organizationId : undefined + ); const [searchValue, setSearchValue] = useState(""); const deferredSearchValue = useDeferredValue(searchValue); const [selectedRow, setSelectedRow] = useState(null); @@ -66,7 +72,7 @@ export const AddUserModal: React.FC<{ closeModal: () => void }> = ({ closeModal scopeType: "workspace", scopeId: workspaceId, }); - closeModal(); + onSubmit(); }; /* Before the user begins typing an email address, the list of users should only be users @@ -75,7 +81,7 @@ export const AddUserModal: React.FC<{ closeModal: () => void }> = ({ closeModal When they begin typing, we filter a list that is a superset of workspaceAccessUsers + organization users. We want to prefer the workspaceAccessUsers object for a given user (if present) because it contains all relevant permissions for the user. - Then, we enrich that from the list of organization_member who don't have a permission to this workspace. + Then, we enrich that from the list of organization_members who don't have a permission to this workspace. */ const userMap = new Map(); @@ -96,9 +102,10 @@ export const AddUserModal: React.FC<{ closeModal: () => void }> = ({ closeModal }); users.forEach((user) => { - // the first check here is important only for the "empty search value" case, where we want to show all users who don't have a workspace permission - // for other cases, it is at worst slightly redundant - if (user.permissionType === "organization_member" && !userMap.has(user.userId)) { + if ( + user.permissionType === "organization_member" && // they are an organization_member + !usersWithAccess.some((u) => u.userId === user.userId) // they don't have a workspace permission (they may not be listed) + ) { userMap.set(user.userId, { userId: user.userId, userName: user.name, diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModalBody.module.scss b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModalBody.module.scss index 141d903c6d6..ab72a4aa61f 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModalBody.module.scss +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModalBody.module.scss @@ -1,5 +1,8 @@ +@use "scss/variables"; + .addUserModalBody { height: 400px; + padding: variables.$spacing-md; &__list, &__listItem { diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModalBody.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModalBody.tsx index 608a2ddeafb..a82388c43ce 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModalBody.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/AddUserModalBody.tsx @@ -7,7 +7,7 @@ import { ModalBody } from "components/ui/Modal"; import { Text } from "components/ui/Text"; import { useCurrentOrganizationInfo } from "core/api"; -import { WorkspaceUserAccessInfoRead } from "core/api/types/AirbyteClient"; +import { PermissionType, WorkspaceUserAccessInfoRead } from "core/api/types/AirbyteClient"; import { AddUserFormValues } from "./AddUserModal"; import styles from "./AddUserModalBody.module.scss"; @@ -34,22 +34,21 @@ export const AddUserModalBody: React.FC = ({ // handle when the selected option is no longer visible useEffect(() => { - // user had selected to invite a new user, then changed the search value so that option is no longer valid, clear form value - if (selectedRow === "inviteNewUser" && !showInviteNewUser) { - setSelectedRow(null); - setValue("email", "", { shouldValidate: true }); - } + const resetPredicates = [ + // user had selected to invite a new user, then changed the search value so that option is no longer valid + selectedRow === "inviteNewUser" && !showInviteNewUser, - // user had selected to invite a new user, then changed the search value to another valid option, clear form value and deselect - if (selectedRow === "inviteNewUser" && deferredSearchValue !== getValues("email")) { - setSelectedRow(null); - setValue("email", "", { shouldValidate: true }); - } + // user had selected to invite a new user, then changed the search value to another valid email + selectedRow === "inviteNewUser" && deferredSearchValue !== getValues("email"), + + // user had selected a user and that user is no longer visible + selectedRow && selectedRow !== "inviteNewUser" && !usersToList.find((user) => user.userId === selectedRow), + ]; - // user had selected a user and that user is no longer visible, clear it - if (selectedRow && selectedRow !== "inviteNewUser" && !usersToList.find((user) => user.userId === selectedRow)) { + if (resetPredicates.some(Boolean)) { setSelectedRow(null); setValue("email", "", { shouldValidate: true }); + setValue("permission", PermissionType.workspace_admin, { shouldValidate: true }); } }, [usersToList, showInviteNewUser, selectedRow, setSelectedRow, setValue, deferredSearchValue, getValues]); diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/DeleteCloudWorkspace.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/DeleteCloudWorkspace.tsx index 5d633d1bc8e..9ed35fb1767 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/DeleteCloudWorkspace.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/DeleteCloudWorkspace.tsx @@ -1,21 +1,50 @@ import React from "react"; -import { FormattedMessage } from "react-intl"; +import { FormattedMessage, useIntl } from "react-intl"; +import { useNavigate } from "react-router-dom"; import { Button } from "components/ui/Button"; -import { useConfirmWorkspaceDeletionModal } from "area/workspace/utils/useConfirmWorkspaceDeletionModal"; import { useCurrentWorkspace } from "core/api"; import { useRemoveCloudWorkspace } from "core/api/cloud"; +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; +import { useNotificationService } from "hooks/services/Notification"; +import { RoutePaths } from "pages/routePaths"; export const DeleteCloudWorkspace: React.FC = () => { const workspace = useCurrentWorkspace(); const { mutateAsync: removeCloudWorkspace, isLoading: isRemovingCloudWorkspace } = useRemoveCloudWorkspace(); + const { registerNotification } = useNotificationService(); + const navigate = useNavigate(); + const { formatMessage } = useIntl(); + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); - const confirmWorkspaceDeletion = useConfirmWorkspaceDeletionModal(workspace, removeCloudWorkspace); + const onRemoveWorkspaceClick = () => + openConfirmationModal({ + text: `settings.workspaceSettings.deleteWorkspace.confirmation.text`, + title: ( + + ), + submitButtonText: "settings.workspaceSettings.delete.confirmation.submitButtonText", + confirmationText: workspace.name, + onSubmit: async () => { + await removeCloudWorkspace(workspace.workspaceId); + registerNotification({ + id: "settings.workspace.delete.success", + text: formatMessage({ id: "settings.workspaceSettings.delete.success" }), + type: "success", + }); + navigate(`/${RoutePaths.Workspaces}`); + closeConfirmationModal(); + }, + submitButtonDataId: "reset", + }); return ( - ); diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/FirebaseInviteUserButton.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/FirebaseInviteUserButton.tsx index 278552e9507..3688db57591 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/FirebaseInviteUserButton.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/FirebaseInviteUserButton.tsx @@ -14,9 +14,11 @@ export const FirebaseInviteUserButton: React.FC = () => { const canUpdateWorkspacePermissions = useIntent("UpdateWorkspacePermissions", { workspaceId }); const onOpenInviteUsersModal = () => - openModal({ + openModal({ title: formatMessage({ id: "modals.addUser.title" }), - content: () => , + content: ({ onComplete, onCancel }) => ( + + ), size: "md", }); diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/InviteUserRow.module.scss b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/InviteUserRow.module.scss index 29da1fcf0aa..3365de164c9 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/InviteUserRow.module.scss +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/InviteUserRow.module.scss @@ -6,28 +6,46 @@ border-bottom: variables.$border-thin solid colors.$grey-50; &__label { + padding: variables.$spacing-md 0; display: inline-block; width: 100%; cursor: pointer; + padding-left: variables.$spacing-md; + padding-right: variables.$spacing-md; + height: 60px; + + &:hover { + background-color: colors.$grey-50; + } + } + + &__labelContent { + height: 100%; } &__dot { flex: 0 0 auto; + line-height: 0; } &__hiddenInput { @include mixins.visually-hidden; - &:checked { - + .radioButtonTiles__toggle { - border-color: colors.$blue; - } - } - &:focus-visible { - + .radioButtonTiles__toggle { - outline: 2px solid colors.$blue-900; + + .inviteUserRow__label { + outline: variables.$border-thin solid colors.$blue-900; } } } + + &__listBoxButton { + border: none; + border-radius: variables.$border-radius-sm; + background-color: transparent; + cursor: pointer; + } + + &__listBoxButton:hover { + background-color: colors.$grey-100; + } } diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/InviteUserRow.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/InviteUserRow.tsx index 1cfcd4b005b..92292df2120 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/InviteUserRow.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/InviteUserRow.tsx @@ -3,19 +3,28 @@ import { useFormContext } from "react-hook-form"; import { FormattedMessage, useIntl } from "react-intl"; import { SelectedIndicatorDot } from "components/connection/CreateConnection/SelectedIndicatorDot"; -import { Badge } from "components/ui/Badge"; import { Box } from "components/ui/Box"; -import { FlexContainer } from "components/ui/Flex"; +import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; +import { ListBox } from "components/ui/ListBox"; import { Text } from "components/ui/Text"; -import { Tooltip } from "components/ui/Tooltip"; import { PermissionType, WorkspaceUserAccessInfoRead } from "core/api/types/AirbyteClient"; import { useCurrentUser } from "core/services/auth"; -import { getWorkspaceAccessLevel } from "pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData"; +import { FeatureItem, useFeature } from "core/services/features"; +import { partitionPermissionType } from "core/utils/rbac/rbacPermissionsQuery"; +import { + getWorkspaceAccessLevel, + permissionsByResourceType, + unifyWorkspaceUserData, +} from "pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData"; +import { UserRoleText } from "pages/SettingsPage/pages/AccessManagementPage/components/UserRoleText"; +import { disallowedRoles } from "pages/SettingsPage/pages/AccessManagementPage/next/ChangeRoleMenuItem"; +import { ChangeRoleMenuItemContent } from "pages/SettingsPage/pages/AccessManagementPage/next/ChangeRoleMenuItemContent"; import { AddUserFormValues } from "./AddUserModal"; -import { ExistingUserIndicator } from "./ExistingUserIndicator"; import styles from "./InviteUserRow.module.scss"; +import { ViewOnlyUserRow } from "./ViewOnlyUserRow"; interface InviteUserRowProps { id: string; @@ -27,62 +36,44 @@ interface InviteUserRowProps { } export const InviteUserRow: React.FC = ({ id, name, email, selectedRow, setSelectedRow, user }) => { - const [permissionType] = useState(PermissionType.workspace_admin); + const transformedUser = !!user ? unifyWorkspaceUserData([user], [])[0] : null; + const allowAllRBACRoles = useFeature(FeatureItem.AllowAllRBACRoles); + + const [selectedPermissionType, setPermissionType] = useState(PermissionType.workspace_admin); const { setValue } = useFormContext(); const { formatMessage } = useIntl(); const { userId: currentUserId } = useCurrentUser(); + const isCurrentUser = user?.userId === currentUserId; + const isOrgAdmin = user?.organizationPermission?.permissionType === PermissionType.organization_admin; const onSelectRow = () => { setSelectedRow(id); - setValue("permission", permissionType, { shouldValidate: true }); + setValue("permission", selectedPermissionType, { shouldValidate: true }); setValue("email", email, { shouldValidate: true }); }; + const onSelectPermission = (selectedValue: PermissionType) => { + setPermissionType(selectedValue); + setValue("permission", selectedValue, { shouldValidate: true }); + }; + const shouldDisableRow = useMemo(() => { - return id === "inviteNewUser" - ? false - : user?.organizationPermission?.permissionType === PermissionType.organization_admin || - !!user?.workspacePermission?.permissionType || - user?.userId === currentUserId; - }, [currentUserId, id, user]); + return id === "inviteNewUser" ? false : isOrgAdmin || !!user?.workspacePermission?.permissionType || isCurrentUser; + }, [id, isOrgAdmin, isCurrentUser, user]); const highestPermissionType = user ? getWorkspaceAccessLevel(user) : undefined; - if (shouldDisableRow && highestPermissionType) { + const selectedPermissionTypeString = partitionPermissionType(selectedPermissionType)[1]; + + if (shouldDisableRow) { return ( - - - - - {name} - {user?.userId === currentUserId && ( - - - - - - )} - - - {email} - - - {user?.organizationPermission?.permissionType === PermissionType.organization_admin ? ( - - - - } - placement="top-start" - > - - - ) : ( - - )} - - + ); } @@ -101,21 +92,63 @@ export const InviteUserRow: React.FC = ({ id, name, email, s {/* the linter cannot seem to keep track of the input + label here */} {/* eslint-disable-next-line jsx-a11y/label-has-associated-control */} ); diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/UpdateCloudWorkspaceName.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/UpdateCloudWorkspaceName.tsx index bea3b49fb93..6757a5828fb 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/UpdateCloudWorkspaceName.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/UpdateCloudWorkspaceName.tsx @@ -75,7 +75,7 @@ export const UpdateCloudWorkspaceName: React.FC = () => { id: "settings.workspaceSettings.updateWorkspaceNameForm.name.placeholder", })} /> - {canUpdateWorkspace && } + {canUpdateWorkspace && } ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/ViewOnlyUserRow.module.scss b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/ViewOnlyUserRow.module.scss new file mode 100644 index 00000000000..ffb765db2d0 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/ViewOnlyUserRow.module.scss @@ -0,0 +1,13 @@ +@use "scss/variables"; +@use "scss/colors"; + +.existingUserRow { + padding-left: variables.$spacing-md; + padding-right: variables.$spacing-md; + height: 60px; + border-bottom: variables.$border-thin solid colors.$grey-50; + + &__content { + height: 100%; + } +} diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/ViewOnlyUserRow.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/ViewOnlyUserRow.tsx new file mode 100644 index 00000000000..8675ee7e84c --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/ViewOnlyUserRow.tsx @@ -0,0 +1,62 @@ +import { FormattedMessage } from "react-intl"; + +import { Badge } from "components/ui/Badge"; +import { Box } from "components/ui/Box"; +import { FlexContainer } from "components/ui/Flex"; +import { Text } from "components/ui/Text"; +import { Tooltip } from "components/ui/Tooltip"; + +import { ExistingUserIndicator } from "./ExistingUserIndicator"; +import styles from "./ViewOnlyUserRow.module.scss"; + +interface ViewOnlyUserRowProps { + name?: string; + email: string; + isCurrentUser: boolean; + isOrgAdmin: boolean; + highestPermissionType?: "ADMIN" | "EDITOR" | "READER" | "MEMBER"; +} +export const ViewOnlyUserRow: React.FC = ({ + name, + email, + isCurrentUser, + isOrgAdmin, + highestPermissionType, +}) => { + return ( + + + + + {name} + {isCurrentUser && ( + + + + + + )} + + + {email} + + + {isOrgAdmin && ( + + + + } + placement="top-start" + > + + + )} + {!isOrgAdmin && !!highestPermissionType && ( + + )} + + + ); +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx index f41894dd430..fc1996ec9fe 100644 --- a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx @@ -1,9 +1,9 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; +import { Separator } from "components/ui/Separator"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { FeatureItem, useFeature } from "core/services/features"; @@ -17,16 +17,16 @@ export const GeneralOrganizationSettingsPage: React.FC = () => { return ( - - + + - - - + + {isAccessManagementEnabled && ( - + <> + - + )} ); diff --git a/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx index 25024a9cfa2..d4225fd78cb 100644 --- a/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx @@ -1,7 +1,6 @@ import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; -import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; @@ -21,32 +20,26 @@ export const GeneralWorkspaceSettingsPage = () => { return ( - + - - - + {isAccessManagementEnabled && ( - - - - - + + + )} {canDeleteWorkspace && ( - - - - - - - - + + + + + + - + )} ); diff --git a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx index 583b2b39372..1268c2c0748 100644 --- a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx @@ -1,14 +1,11 @@ import React, { Suspense } from "react"; -import { FormattedMessage, useIntl } from "react-intl"; +import { useIntl } from "react-intl"; import { Outlet } from "react-router-dom"; -import { LoadingPage, MainPageWithScroll } from "components"; -import { HeadTitle } from "components/common/HeadTitle"; -import { SettingsLink, SettingsNavigation, SettingsNavigationBlock } from "components/settings/SettingsNavigation"; -import { FlexContainer, FlexItem } from "components/ui/Flex"; -import { Heading } from "components/ui/Heading"; -import { PageHeader } from "components/ui/PageHeader"; +import { LoadingPage } from "components"; +import { SettingsLayout, SettingsLayoutContent } from "area/settings/components/SettingsLayout"; +import { SettingsLink, SettingsNavigation, SettingsNavigationBlock } from "area/settings/components/SettingsNavigation"; import { useCurrentWorkspace, useGetInstanceConfiguration } from "core/api"; import { InstanceConfigurationResponseTrackingStrategy } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; @@ -27,110 +24,97 @@ export const SettingsPage: React.FC = () => { const { formatMessage } = useIntl(); return ( - } - pageTitle={ - - - - } - /> - } - > - - - + + + + + {apiTokenManagement && ( - {apiTokenManagement && ( + )} + + {canViewWorkspaceSettings && ( + + {multiWorkspaceUI && ( )} - - {canViewWorkspaceSettings && ( - - {multiWorkspaceUI && ( + {!multiWorkspaceUI && ( + <> - )} - {!multiWorkspaceUI && ( - <> - - - - )} + + + )} + + {trackingStrategy === InstanceConfigurationResponseTrackingStrategy.segment && ( + + )} + + )} + {multiWorkspaceUI && (canViewOrganizationSettings || canViewWorkspaceSettings) && ( + + {multiWorkspaceUI && canViewOrganizationSettings && ( - {trackingStrategy === InstanceConfigurationResponseTrackingStrategy.segment && ( + )} + {multiWorkspaceUI && canViewWorkspaceSettings && ( + <> - )} - - )} - {multiWorkspaceUI && (canViewOrganizationSettings || canViewWorkspaceSettings) && ( - - {multiWorkspaceUI && canViewOrganizationSettings && ( - )} - {multiWorkspaceUI && canViewWorkspaceSettings && ( - <> - - - - )} - - )} - - - }> - - - - - + + )} + + )} + + + }> + + + + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/UpdateOrganizationSettingsForm.tsx b/airbyte-webapp/src/pages/SettingsPage/UpdateOrganizationSettingsForm.tsx index f360ee2a6bf..967d3258b9d 100644 --- a/airbyte-webapp/src/pages/SettingsPage/UpdateOrganizationSettingsForm.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/UpdateOrganizationSettingsForm.tsx @@ -80,7 +80,7 @@ const OrganizationSettingsForm = ({ organizationId }: { organizationId: string } name="email" labelTooltip={formatMessage({ id: "settings.organizationSettings.email.description" })} /> - {canUpdateOrganization && } + {canUpdateOrganization && } ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx b/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx index 94f426614a1..2d4dbfb8313 100644 --- a/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx @@ -1,17 +1,47 @@ -import { FormattedMessage } from "react-intl"; +import { FormattedMessage, useIntl } from "react-intl"; +import { useNavigate } from "react-router-dom"; import { Button } from "components/ui/Button"; -import { useConfirmWorkspaceDeletionModal } from "area/workspace/utils/useConfirmWorkspaceDeletionModal"; import { useCurrentWorkspace, useDeleteWorkspace } from "core/api"; +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; +import { useNotificationService } from "hooks/services/Notification"; +import { RoutePaths } from "pages/routePaths"; export const DeleteWorkspace: React.FC = () => { const workspace = useCurrentWorkspace(); const { mutateAsync: deleteWorkspace, isLoading: isDeletingWorkspace } = useDeleteWorkspace(); - const confirmWorkspaceDeletion = useConfirmWorkspaceDeletionModal(workspace, deleteWorkspace); + const { registerNotification } = useNotificationService(); + const navigate = useNavigate(); + const { formatMessage } = useIntl(); + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); + + const onRemoveWorkspaceClick = () => + openConfirmationModal({ + text: `settings.workspaceSettings.deleteWorkspace.confirmation.text`, + title: ( + + ), + submitButtonText: "settings.workspaceSettings.delete.confirmation.submitButtonText", + confirmationText: workspace.name, + onSubmit: async () => { + await deleteWorkspace(workspace.workspaceId); + registerNotification({ + id: "settings.workspace.delete.success", + text: formatMessage({ id: "settings.workspaceSettings.delete.success" }), + type: "success", + }); + navigate(`/${RoutePaths.Workspaces}`); + closeConfirmationModal(); + }, + submitButtonDataId: "reset", + }); return ( - ); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementSection.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementSection.tsx index 6093071c655..867bf2a8469 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementSection.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementSection.tsx @@ -5,6 +5,7 @@ import { useSearchParams } from "react-router-dom"; import { Badge } from "components/ui/Badge"; import { Box } from "components/ui/Box"; import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; import { Icon } from "components/ui/Icon"; import { ExternalLink } from "components/ui/Link"; import { SearchInput } from "components/ui/SearchInput"; @@ -49,9 +50,9 @@ export const OrganizationAccessManagementSection: React.FC = () => { return ( - + - + diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationUsersTable.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationUsersTable.tsx index fe8d42d581e..52a8a31c26a 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationUsersTable.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationUsersTable.tsx @@ -76,5 +76,5 @@ export const OrganizationUsersTable: React.FC<{ [columnHelper, currentUserId] ); - return
; + return
; }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementSection.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementSection.tsx index 035aa1cf362..7ebe302cb97 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementSection.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementSection.tsx @@ -5,6 +5,7 @@ import { useSearchParams } from "react-router-dom"; import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; import { SearchInput } from "components/ui/SearchInput"; import { Text } from "components/ui/Text"; @@ -21,7 +22,7 @@ import { AddUserModal } from "packages/cloud/views/workspaces/WorkspaceSettingsV import { FirebaseInviteUserButton } from "packages/cloud/views/workspaces/WorkspaceSettingsView/components/FirebaseInviteUserButton"; import { AddUserControl } from "./components/AddUserControl"; -import { unifyWorkspaceUserData, UnifiedWorkspaceUserModel } from "./components/useGetAccessManagementData"; +import { UnifiedWorkspaceUserModel, unifyWorkspaceUserData } from "./components/useGetAccessManagementData"; import styles from "./WorkspaceAccessManagementSection.module.scss"; import { WorkspaceUsersTable } from "./WorkspaceUsersTable"; @@ -32,7 +33,7 @@ const WorkspaceAccessManagementSection: React.FC = () => { const organization = useCurrentOrganizationInfo(); const canViewOrgMembers = useIntent("ListOrganizationMembers", { organizationId: organization?.organizationId }); const canUpdateWorkspacePermissions = useIntent("UpdateWorkspacePermissions", { workspaceId: workspace.workspaceId }); - const { openModal, closeModal } = useModalService(); + const { openModal } = useModalService(); const usersWithAccess = useListWorkspaceAccessUsers(workspace.workspaceId).usersWithAccess; @@ -53,9 +54,9 @@ const WorkspaceAccessManagementSection: React.FC = () => { const invitationSystemv2 = useExperiment("settings.invitationSystemv2", false); const onOpenInviteUsersModal = () => - openModal({ + openModal({ title: formatMessage({ id: "userInvitations.create.modal.title" }, { workspace: workspace.name }), - content: () => , + content: ({ onComplete }) => , size: "md", }); @@ -78,9 +79,9 @@ const WorkspaceAccessManagementSection: React.FC = () => { return ( - + - + diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceUsersTable.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceUsersTable.tsx index a92c6ad3173..67067535bce 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceUsersTable.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceUsersTable.tsx @@ -72,5 +72,7 @@ export const WorkspaceUsersTable: React.FC<{ [areAllRbacRolesEnabled, columnHelper, currentUserId] ); - return
; + return ( +
+ ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AddUserControl.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AddUserControl.tsx index a32d0817d83..fc498ea4d32 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AddUserControl.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AddUserControl.tsx @@ -103,6 +103,7 @@ const AddUserForm: React.FC<{ submitKey="form.add" onCancelClickCallback={() => setIsEditMode(false)} allowNonDirtyCancel + allowNonDirtySubmit /> diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/UserRoleText.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/UserRoleText.tsx index bfbe662acb3..f45c150867a 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/UserRoleText.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/UserRoleText.tsx @@ -19,7 +19,7 @@ export const UserRoleText: React.FC<{ highestPermissionType?: RbacRole }> = ({ h : "role.member"; return ( - + ); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx index 4c890277858..ecd88592477 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx @@ -85,7 +85,7 @@ export const unifyWorkspaceUserData = ( const normalizedInvitations = workspaceInvitations.map((invitation) => { return { - id: invitation.id, + id: invitation.inviteCode, userEmail: invitation.invitedEmail, invitationStatus: invitation.status, invitationPermissionType: invitation.permissionType, diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/CancelInvitationMenuItem.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/CancelInvitationMenuItem.tsx new file mode 100644 index 00000000000..89ed904d1ac --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/CancelInvitationMenuItem.tsx @@ -0,0 +1,60 @@ +import { FormattedMessage } from "react-intl"; + +import { Box } from "components/ui/Box"; +import { Text } from "components/ui/Text"; + +import { useCancelUserInvitation, useCurrentWorkspace } from "core/api"; +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; + +import styles from "./RemoveRoleMenuItem.module.scss"; +import { UnifiedWorkspaceUserModel } from "../components/useGetAccessManagementData"; + +interface CancelInvitationMenuItemProps { + user: UnifiedWorkspaceUserModel; +} + +export const CancelInvitationMenuItem: React.FC = ({ user }) => { + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); + + const { name: workspaceName } = useCurrentWorkspace(); + + const { mutateAsync: cancelInvitation } = useCancelUserInvitation(); + + const onClick = () => + openConfirmationModal({ + text: ( + + {user.userEmail} + + ), + resource: ( + + {workspaceName} + + ), + }} + /> + ), + title: , + submitButtonText: "userInvitations.cancel.confirm.title", + onSubmit: async () => { + await cancelInvitation({ inviteCode: user.id }); + closeConfirmationModal(); + }, + submitButtonDataId: "cancel-invite", + }); + + return ( + + ); +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/ChangeRoleMenuItem.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/ChangeRoleMenuItem.tsx index b93ea5b7a7a..92d109e4bbb 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/ChangeRoleMenuItem.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/ChangeRoleMenuItem.tsx @@ -1,24 +1,15 @@ import classNames from "classnames"; -import { FormattedMessage } from "react-intl"; - -import { Box } from "components/ui/Box"; -import { FlexContainer, FlexItem } from "components/ui/Flex"; -import { Icon } from "components/ui/Icon"; -import { Text } from "components/ui/Text"; import { useCreatePermission, useCurrentOrganizationInfo, useCurrentWorkspace, useUpdatePermissions } from "core/api"; -import { PermissionType, WorkspaceUserAccessInfoRead } from "core/api/types/AirbyteClient"; +import { PermissionType } from "core/api/types/AirbyteClient"; import { useCurrentUser } from "core/services/auth"; import styles from "./ChangeRoleMenuItem.module.scss"; -import { - ResourceType, - permissionStringDictionary, - permissionDescriptionDictionary, -} from "../components/useGetAccessManagementData"; +import { ChangeRoleMenuItemContent } from "./ChangeRoleMenuItemContent"; +import { ResourceType, UnifiedWorkspaceUserModel } from "../components/useGetAccessManagementData"; const useCreateOrUpdateRole = ( - user: WorkspaceUserAccessInfoRead, + user: UnifiedWorkspaceUserModel, resourceType: ResourceType, permissionType: PermissionType ) => { @@ -41,12 +32,12 @@ const useCreateOrUpdateRole = ( throw new Error("Organization info not found"); } return createPermission({ - userId: user.userId, + userId: user.id, permissionType, organizationId: organizationInfo.organizationId, }); } - return createPermission({ userId: user.userId, permissionType, workspaceId }); + return createPermission({ userId: user.id, permissionType, workspaceId }); } return updatePermission({ permissionId: existingPermissionIdForResourceType, permissionType }); @@ -54,7 +45,7 @@ const useCreateOrUpdateRole = ( }; export const disallowedRoles = ( - user: WorkspaceUserAccessInfoRead, + user: UnifiedWorkspaceUserModel | null, targetResourceType: ResourceType, isCurrentUser: boolean ): PermissionType[] => { @@ -74,7 +65,7 @@ export const disallowedRoles = ( return []; } - const organizationRole = user.organizationPermission?.permissionType; + const organizationRole = user?.organizationPermission?.permissionType; if (organizationRole === "organization_editor") { return ["workspace_reader"]; @@ -87,7 +78,7 @@ export const disallowedRoles = ( }; interface RoleMenuItemProps { - user: WorkspaceUserAccessInfoRead; + user: UnifiedWorkspaceUserModel; permissionType: PermissionType; resourceType: ResourceType; onClose: () => void; @@ -96,7 +87,7 @@ interface RoleMenuItemProps { export const ChangeRoleMenuItem: React.FC = ({ user, permissionType, resourceType, onClose }) => { const createOrUpdateRole = useCreateOrUpdateRole(user, resourceType, permissionType); const currentUser = useCurrentUser(); - const isCurrentUser = currentUser.userId === user.userId; + const isCurrentUser = currentUser.userId === user.id; const roleIsActive = permissionType === user.workspacePermission?.permissionType || @@ -115,26 +106,11 @@ export const ChangeRoleMenuItem: React.FC = ({ user, permissi [styles["changeRoleMenuItem__button--active"]]: roleIsActive, })} > - - - - - - - - - - - {roleIsActive && ( - - - - )} - - + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/ChangeRoleMenuItemContent.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/ChangeRoleMenuItemContent.tsx new file mode 100644 index 00000000000..8af24f943f2 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/ChangeRoleMenuItemContent.tsx @@ -0,0 +1,45 @@ +import { FormattedMessage } from "react-intl"; + +import { Box } from "components/ui/Box"; +import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; +import { Text } from "components/ui/Text"; + +import { PermissionType } from "core/api/types/AirbyteClient"; + +import { permissionDescriptionDictionary, permissionStringDictionary } from "../components/useGetAccessManagementData"; + +interface ChangeRoleMenuItemContentProps { + roleIsInvalid: boolean; + roleIsActive: boolean; + permissionType: PermissionType; +} + +export const ChangeRoleMenuItemContent: React.FC = ({ + roleIsActive, + permissionType, + roleIsInvalid, +}) => { + return ( + + + + + + + + + + + {roleIsActive && ( + + + + )} + + + ); +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RemoveRoleMenuItem.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RemoveRoleMenuItem.tsx index 4b753ea1474..f37f3f9902c 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RemoveRoleMenuItem.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RemoveRoleMenuItem.tsx @@ -4,15 +4,14 @@ import { Box } from "components/ui/Box"; import { Text } from "components/ui/Text"; import { useCurrentOrganizationInfo, useCurrentWorkspace, useDeletePermissions } from "core/api"; -import { WorkspaceUserAccessInfoRead } from "core/api/types/AirbyteClient"; import { useCurrentUser } from "core/services/auth"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import styles from "./RemoveRoleMenuItem.module.scss"; -import { ResourceType } from "../components/useGetAccessManagementData"; +import { ResourceType, UnifiedWorkspaceUserModel } from "../components/useGetAccessManagementData"; interface RemoveRoleMenuItemProps { - user: WorkspaceUserAccessInfoRead; + user: UnifiedWorkspaceUserModel; resourceType: ResourceType; } @@ -53,11 +52,11 @@ export const RemoveRoleMenuItem: React.FC = ({ user, re return ( + ); +}); + +RoleManagementButton.displayName = "RoleManagementButton"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementCell.tsx index 526b494ef17..84d3e997540 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementCell.tsx @@ -50,8 +50,7 @@ export const RoleManagementCell: React.FC = ({ user, re organizationId: organizationInfo?.organizationId, }); const cannotDemoteUser = resourceType === "workspace" && orgPermissionType === "organization_admin"; - const shouldHidePopover = - cannotDemoteUser || !canEditPermissions || user.id === currentUser.userId || user.invitationStatus === "pending"; + const shouldHidePopover = cannotDemoteUser || !canEditPermissions || user.id === currentUser.userId; const tooltipContent = cannotDemoteUser && canEditPermissions @@ -71,10 +70,7 @@ export const RoleManagementCell: React.FC = ({ user, re ) ) : ( - + )} {user.organizationPermission?.permissionType === "organization_admin" && resourceType === "workspace" && ( diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenu.module.scss b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenu.module.scss index ca2ac525d75..9c5b213584a 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenu.module.scss +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenu.module.scss @@ -2,17 +2,6 @@ @use "scss/variables"; @use "scss/z-indices"; -.roleManagementMenu__popoverButton { - border: none; - border-radius: variables.$border-radius-sm; - background-color: transparent; - cursor: pointer; -} - -.roleManagementMenu__popoverButton:hover { - background-color: colors.$grey-100; -} - .roleManagementMenu__popoverPanel { position: relative; z-index: z-indices.$dropdownMenu; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenu.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenu.tsx index e6527ae443a..4ecb67402bb 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenu.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenu.tsx @@ -6,39 +6,32 @@ import { Box } from "components/ui/Box"; import { FlexContainer, FlexItem } from "components/ui/Flex"; import { Icon } from "components/ui/Icon"; -import { WorkspaceUserAccessInfoRead } from "core/api/types/AirbyteClient"; +import { RbacRole } from "core/utils/rbac/rbacPermissionsQuery"; +import { RoleManagementButton } from "./RoleManagementButton"; import styles from "./RoleManagementMenu.module.scss"; import { RoleManagementMenuBody } from "./RoleManagementMenuBody"; -import { getWorkspaceAccessLevel } from "../components/useGetAccessManagementData"; +import { UnifiedWorkspaceUserModel } from "../components/useGetAccessManagementData"; import { UserRoleText } from "../components/UserRoleText"; type ResourceType = "workspace" | "organization" | "instance"; export interface RoleManagementMenuProps { - user: WorkspaceUserAccessInfoRead; + user: UnifiedWorkspaceUserModel; resourceType: ResourceType; + highestPermissionType: RbacRole; } -const RoleManagementButton = React.forwardRef>( - ({ children, ...props }, ref) => { - return ( - - ); - } -); - -RoleManagementButton.displayName = "RoleManagementButton"; - -export const RoleManagementMenu: React.FC = ({ user, resourceType }) => { +export const RoleManagementMenu: React.FC = ({ + user, + resourceType, + highestPermissionType, +}) => { const { x, y, reference, floating, strategy } = useFloating({ middleware: [offset(5), flip()], whileElementsMounted: autoUpdate, placement: "bottom-start", }); - const highestPermissionType = getWorkspaceAccessLevel(user); return ( diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenuBody.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenuBody.tsx index 70bc897e5dc..0773c716d22 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenuBody.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/next/RoleManagementMenuBody.tsx @@ -3,19 +3,20 @@ import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; import { Text } from "components/ui/Text"; -import { WorkspaceUserAccessInfoRead } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; +import { CancelInvitationMenuItem } from "./CancelInvitationMenuItem"; import { ChangeRoleMenuItem } from "./ChangeRoleMenuItem"; import { RemoveRoleMenuItem } from "./RemoveRoleMenuItem"; import styles from "./RoleManagementMenuBody.module.scss"; import { ResourceType, + UnifiedWorkspaceUserModel, permissionStringDictionary, permissionsByResourceType, } from "../components/useGetAccessManagementData"; interface RoleManagementMenuBodyProps { - user: WorkspaceUserAccessInfoRead; + user: UnifiedWorkspaceUserModel; resourceType: ResourceType; close: () => void; } @@ -27,13 +28,15 @@ export const RoleManagementMenuBody: React.FC = ({ */ const rolesToAllow = - areAllRbacRolesEnabled || resourceType === "organization" ? permissionsByResourceType[resourceType] : []; + !user.invitationStatus && (areAllRbacRolesEnabled || resourceType === "organization") + ? permissionsByResourceType[resourceType] + : []; return (
    {resourceType === "workspace" && - user.organizationPermission?.permissionType && - user.organizationPermission?.permissionType !== "organization_member" && ( + user?.organizationPermission?.permissionType && + user?.organizationPermission?.permissionType !== "organization_member" && (
  • = ({
    • ); })} - {resourceType === "workspace" && ( + {resourceType === "workspace" && !!user.invitationStatus && ( +
  • + +
    • + )} + {resourceType === "workspace" && !user.invitationStatus && (
    • diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/AccountPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/AccountPage.tsx index 7b92866f995..017d88567bc 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/AccountPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/AccountPage.tsx @@ -1,12 +1,8 @@ -import React, { useState } from "react"; -import { FormattedMessage, useIntl } from "react-intl"; -import { useAuth } from "react-oidc-context"; +import React from "react"; +import { useIntl } from "react-intl"; -import { HeadTitle } from "components/common/HeadTitle"; -import { Box } from "components/ui/Box"; -import { Button } from "components/ui/Button"; -import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; import { FeatureItem, useFeature } from "core/services/features"; @@ -18,36 +14,9 @@ export const AccountPage: React.FC = () => { const isKeycloakAuthenticationEnabled = useFeature(FeatureItem.KeycloakAuthentication); return ( - <> - - - {isKeycloakAuthenticationEnabled ? : } - - {isKeycloakAuthenticationEnabled && } - - ); -}; - -const SignoutButton: React.FC = () => { - const [signoutRedirectPending, setSignnoutRedirectPending] = useState(false); - const auth = useAuth(); - - const handleSignout = () => { - setSignnoutRedirectPending(true); - auth.signoutRedirect(); - }; - return ( - - - - + + {formatMessage({ id: "settings.accountSettings" })} + {isKeycloakAuthenticationEnabled ? : } ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx index 4555fc34060..c4b91d608ea 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx @@ -56,7 +56,7 @@ export const AccountForm: React.FC = () => { defaultValues={{ email: workspace.email ?? "" }} > label={formatMessage({ id: "form.yourEmail" })} fieldType="input" name="email" /> - + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AdvancedSettingsPage/AdvancedSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AdvancedSettingsPage/AdvancedSettingsPage.tsx index d96345cace3..8cb75eb4511 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AdvancedSettingsPage/AdvancedSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AdvancedSettingsPage/AdvancedSettingsPage.tsx @@ -1,7 +1,7 @@ import { useIntl } from "react-intl"; -import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; import { Message } from "components/ui/Message"; import { Switch } from "components/ui/Switch"; import { Text } from "components/ui/Text"; @@ -38,24 +38,23 @@ export const AdvancedSettingsPage: React.FC = () => { const [attemptsStats, setAttemptsStats] = useLocalStorage("airbyte_extended-attempts-stats", false); return ( - - - - setWorkspaceInTitle(checked)} - label={formatMessage({ id: "settings.advancedSettings.workspaceInTitle" })} - description={formatMessage({ id: "settings.advancedSettings.workspaceInTitleDescription" })} - /> - setAttemptsStats(checked)} - label={formatMessage({ id: "settings.advancedSettings.attemptStats" })} - description={formatMessage({ id: "settings.advancedSettings.attemptStatsDescription" })} - /> - - + + {formatMessage({ id: "settings.advancedSettings.title" })} + + setWorkspaceInTitle(checked)} + label={formatMessage({ id: "settings.advancedSettings.workspaceInTitle" })} + description={formatMessage({ id: "settings.advancedSettings.workspaceInTitleDescription" })} + /> + setAttemptsStats(checked)} + label={formatMessage({ id: "settings.advancedSettings.attemptStats" })} + description={formatMessage({ id: "settings.advancedSettings.attemptStatsDescription" })} + /> + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/AddCustomDockerImageConnectorModal.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/AddCustomDockerImageConnectorModal.tsx new file mode 100644 index 00000000000..017ee2634cd --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/AddCustomDockerImageConnectorModal.tsx @@ -0,0 +1,96 @@ +import React, { useState } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; +import * as yup from "yup"; + +import { Form, FormControl } from "components/forms"; +import { ModalFormSubmissionButtons } from "components/forms/ModalFormSubmissionButtons"; +import { Box } from "components/ui/Box"; +import { ExternalLink } from "components/ui/Link"; +import { Message } from "components/ui/Message"; +import { ModalBody, ModalFooter } from "components/ui/Modal"; +import { Text } from "components/ui/Text"; + +import { isCloudApp } from "core/utils/app"; +import { links } from "core/utils/links"; + +interface ConnectorDefinition { + name: string; + documentationUrl: string; + dockerImageTag: string; + dockerRepository: string; +} + +export interface AddCustomDockerImageConnectorModalProps { + onCancel: () => void; + onSubmit: (sourceDefinition: ConnectorDefinition) => Promise; +} + +const validationSchema = yup.object().shape({ + name: yup.string().trim().required("form.empty.error"), + documentationUrl: yup.string().trim().url("form.url.error").notRequired().default(""), + dockerImageTag: yup.string().trim().required("form.empty.error"), + dockerRepository: yup.string().trim().required("form.empty.error"), +}); + +const ConnectorControl = FormControl; + +export const AddCustomDockerImageConnectorModal: React.FC = ({ + onCancel, + onSubmit, +}) => { + const { formatMessage } = useIntl(); + const [error, setError] = useState(); + + return ( + + defaultValues={{ + name: "", + documentationUrl: "", + dockerImageTag: "", + dockerRepository: "", + }} + schema={validationSchema} + onSubmit={async (values) => { + setError(undefined); + await onSubmit(values); + }} + onError={(e) => { + setError(e.message || formatMessage({ id: "form.dockerError" })); + }} + > + + + {lnk}, + }} + /> + + + + + + + {error && } + + + + + + + ); +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/AddNewConnectorButton.tsx similarity index 58% rename from airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx rename to airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/AddNewConnectorButton.tsx index c7c30dd6419..0fee33ee727 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/AddNewConnectorButton.tsx @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useNavigate } from "react-router-dom"; @@ -9,38 +9,34 @@ import { Icon } from "components/ui/Icon"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useCreateDestinationDefinition, useCreateSourceDefinition } from "core/api"; import { FeatureItem, useFeature } from "core/services/features"; +import { useModalService } from "hooks/services/Modal"; import { ConnectorBuilderRoutePaths } from "pages/connectorBuilder/ConnectorBuilderRoutes"; import { DestinationPaths, RoutePaths, SourcePaths } from "pages/routePaths"; -import CreateConnectorModal from "./CreateConnectorModal"; +import { AddCustomDockerImageConnectorModal } from "./AddCustomDockerImageConnectorModal"; -interface IProps { +interface AddNewConnectorButtonProps { type: "sources" | "destinations"; } -interface ICreateProps { +interface ConnectorDefinitionProps { name: string; documentationUrl: string; dockerImageTag: string; dockerRepository: string; } -const CreateConnector: React.FC = ({ type }) => { +export const AddNewConnectorButton: React.FC = ({ type }) => { + const { formatMessage } = useIntl(); + const allowUploadCustomDockerImage = useFeature(FeatureItem.AllowUploadCustomImage); const navigate = useNavigate(); const workspaceId = useCurrentWorkspaceId(); - const [isModalOpen, setIsModalOpen] = useState(false); - const onChangeModalState = () => { - setIsModalOpen(!isModalOpen); - }; - const allowUploadCustomImage = useFeature(FeatureItem.AllowUploadCustomImage); - - const { formatMessage } = useIntl(); + const { openModal } = useModalService(); const { mutateAsync: createSourceDefinition } = useCreateSourceDefinition(); - const { mutateAsync: createDestinationDefinition } = useCreateDestinationDefinition(); - const onSubmitSource = async (sourceDefinition: ICreateProps) => { + const onSubmitSource = async (sourceDefinition: ConnectorDefinitionProps) => { const result = await createSourceDefinition(sourceDefinition); navigate({ @@ -48,7 +44,7 @@ const CreateConnector: React.FC = ({ type }) => { }); }; - const onSubmitDestination = async (destinationDefinition: ICreateProps) => { + const onSubmitDestination = async (destinationDefinition: ConnectorDefinitionProps) => { const result = await createDestinationDefinition(destinationDefinition); navigate({ @@ -56,17 +52,33 @@ const CreateConnector: React.FC = ({ type }) => { }); }; - const onSubmit = (values: ICreateProps) => + const onSubmit = (values: ConnectorDefinitionProps) => type === "sources" ? onSubmitSource(values) : onSubmitDestination(values); - if (type === "destinations" && !allowUploadCustomImage) { + const openAddCustomDockerImageConnectorModal = () => + openModal({ + title: formatMessage({ id: "admin.addNewConnector" }), + content: ({ onComplete, onCancel }) => ( + { + await onSubmit(values); + onComplete(); + }} + /> + ), + }); + + if (type === "destinations" && !allowUploadCustomDockerImage) { return null; } return ( <> - {type === "destinations" && allowUploadCustomImage ? ( - + {type === "destinations" && allowUploadCustomDockerImage ? ( + ) : ( = ({ type }) => { displayName: formatMessage({ id: "admin.newConnector.build" }), internal: true, }, - ...(allowUploadCustomImage + ...(allowUploadCustomDockerImage ? [ { as: "button" as const, @@ -89,28 +101,17 @@ const CreateConnector: React.FC = ({ type }) => { ] : []), ]} - onChange={(data: DropdownMenuOptionType) => data.value === "docker" && onChangeModalState()} + onChange={(data: DropdownMenuOptionType) => + data.value === "docker" && openAddCustomDockerImageConnectorModal() + } > - {() => } + {() => ( + + )} )} - - {isModalOpen && } ); }; - -interface NewConnectorButtonProps { - onClick?: () => void; -} - -const NewConnectorButton = React.forwardRef(({ onClick }, ref) => { - return ( - - ); -}); -NewConnectorButton.displayName = "NewConnectorButton"; - -export default CreateConnector; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx index 051798ad904..9ae83f14058 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx @@ -2,7 +2,6 @@ import { createColumnHelper } from "@tanstack/react-table"; import React, { useCallback, useMemo, useState } from "react"; import { FormattedMessage } from "react-intl"; -import { HeadTitle } from "components/common/HeadTitle"; import { ConnectorBuilderProjectTable } from "components/ConnectorBuilderProjectTable"; import { FlexContainer, FlexItem } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; @@ -16,10 +15,10 @@ import { FeatureItem, useFeature } from "core/services/features"; import { useIntent } from "core/utils/rbac"; import { RoutePaths } from "pages/routePaths"; +import { AddNewConnectorButton } from "./AddNewConnectorButton"; import { ConnectorCell } from "./ConnectorCell"; import styles from "./ConnectorsView.module.scss"; import { ConnectorsViewContext } from "./ConnectorsViewContext"; -import CreateConnector from "./CreateConnector"; import ImageCell from "./ImageCell"; import { UpdateDestinationConnectorVersionCell } from "./UpdateDestinationConnectorVersionCell"; import { UpdateSourceConnectorVersionCell } from "./UpdateSourceConnectorVersionCell"; @@ -210,22 +209,31 @@ const ConnectorsView: React.FC = ({ sections.push({ title: type === "sources" ? "admin.manageSource" : "admin.manageDestination", content: ( - + ), }); } sections.push({ title: type === "sources" ? "admin.availableSource" : "admin.availableDestinations", - content: , + content: ( + + ), }); return (
    - {sections.map((section, index) => ( @@ -237,7 +245,7 @@ const ConnectorsView: React.FC = ({ {index === 0 && ( - + {allowUpdateConnectors && } )} diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx deleted file mode 100644 index c3f37435c6b..00000000000 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx +++ /dev/null @@ -1,96 +0,0 @@ -import React, { useState } from "react"; -import { FormattedMessage, useIntl } from "react-intl"; -import * as yup from "yup"; - -import { Form, FormControl } from "components/forms"; -import { ModalFormSubmissionButtons } from "components/forms/ModalFormSubmissionButtons"; -import { Box } from "components/ui/Box"; -import { ExternalLink } from "components/ui/Link"; -import { Message } from "components/ui/Message"; -import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; -import { Text } from "components/ui/Text"; - -import { isCloudApp } from "core/utils/app"; -import { links } from "core/utils/links"; - -interface ConnectorDefinition { - name: string; - documentationUrl: string; - dockerImageTag: string; - dockerRepository: string; -} - -export interface CreateConnectorModalProps { - onClose: () => void; - onSubmit: (sourceDefinition: ConnectorDefinition) => Promise; -} -const validationSchema = yup.object().shape({ - name: yup.string().trim().required("form.empty.error"), - documentationUrl: yup.string().trim().url("form.url.error").notRequired().default(""), - dockerImageTag: yup.string().trim().required("form.empty.error"), - dockerRepository: yup.string().trim().required("form.empty.error"), -}); - -const ConnectorControl = FormControl; - -const CreateConnectorModal: React.FC = ({ onClose, onSubmit }) => { - const { formatMessage } = useIntl(); - const [error, setError] = useState(); - - return ( - }> - - defaultValues={{ - name: "", - documentationUrl: "", - dockerImageTag: "", - dockerRepository: "", - }} - schema={validationSchema} - onSubmit={async (values) => { - setError(undefined); - await onSubmit(values); - }} - onError={(e) => { - setError(e.message || formatMessage({ id: "form.dockerError" })); - }} - > - - - {lnk}, - }} - /> - - - - - - - {error && } - - - - - - - - ); -}; - -export default CreateConnectorModal; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionChangeResult.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionChangeResult.tsx deleted file mode 100644 index 09e4a151549..00000000000 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionChangeResult.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import React from "react"; -import { useFormState } from "react-hook-form"; -import { FormattedMessage } from "react-intl"; - -import { Text } from "components/ui/Text"; - -export const VersionChangeResult: React.FC<{ feedback?: string }> = ({ feedback }) => { - const { isDirty } = useFormState(); - - if (feedback === "success" && !isDirty) { - return ; - } - if (feedback && feedback !== "success") { - return ( - - {feedback} - - ); - } - - return null; -}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/MetricsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/MetricsPage.tsx index b63e6b36543..45063f9cba7 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/MetricsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/MetricsPage.tsx @@ -1,8 +1,8 @@ import React from "react"; import { useIntl } from "react-intl"; -import { HeadTitle } from "components/common/HeadTitle"; -import { Card } from "components/ui/Card"; +import { FlexContainer } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; @@ -13,11 +13,9 @@ export const MetricsPage: React.FC = () => { useTrackPage(PageTrackingCodes.SETTINGS_METRICS); return ( - <> - - - - - + + {formatMessage({ id: "settings.metricsSettings" })} + + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/components/MetricsForm.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/components/MetricsForm.tsx index 0725590584b..886dc16077e 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/components/MetricsForm.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/components/MetricsForm.tsx @@ -71,7 +71,7 @@ export const MetricsForm: React.FC = () => { description={formatMessage({ id: "preferences.collectData" })} /> - + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/NotificationPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/NotificationPage.tsx index 0fca1055212..de2b1fba3d7 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/NotificationPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/NotificationPage.tsx @@ -1,11 +1,10 @@ import React from "react"; import { useIntl } from "react-intl"; -import { HeadTitle } from "components/common/HeadTitle"; import { NotificationSettingsForm } from "components/NotificationSettingsForm"; -import { PageContainer } from "components/PageContainer"; -import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; +import { Separator } from "components/ui/Separator"; import { WorkspaceEmailForm } from "components/WorkspaceEmailForm"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; @@ -17,18 +16,15 @@ export const NotificationPage: React.FC = () => { const emailNotificationsFeatureEnabled = useFeature(FeatureItem.EmailNotifications); return ( - - - - {emailNotificationsFeatureEnabled && ( - - - - )} - - - - - + + {formatMessage({ id: "settings.notificationSettings" })} + {emailNotificationsFeatureEnabled && ( + <> + + + + )} + + ); }; diff --git a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx index 707be83a808..35d591262d6 100644 --- a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx @@ -1,4 +1,4 @@ -import React, { Suspense, useDeferredValue, useMemo, useState } from "react"; +import React, { Suspense, useDeferredValue, useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; @@ -52,13 +52,13 @@ export const AllConnectionsPage: React.FC = () => { const connectionList = useConnectionList(); const connections = useMemo(() => connectionList?.connections ?? [], [connectionList?.connections]); - const [searchFilter, setSearchFilter] = useState(""); - const debouncedSearchFilter = useDeferredValue(searchFilter); const [filterValues, setFilterValue, setFilters] = useFilters({ + search: "", status: null, source: null, destination: null, }); + const debouncedSearchFilter = useDeferredValue(filterValues.search); const filteredConnections = useMemo(() => { const statusFilter = filterValues.status; @@ -185,8 +185,8 @@ export const AllConnectionsPage: React.FC = () => { {isConnectionsSummaryEnabled && ( setFilterValue("search", search)} filterValues={filterValues} setFilterValue={setFilterValue} setFilters={setFilters} diff --git a/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsFilters/ConnectionsFilters.tsx b/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsFilters/ConnectionsFilters.tsx index 0fab06ea030..39da0d21b53 100644 --- a/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsFilters/ConnectionsFilters.tsx +++ b/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsFilters/ConnectionsFilters.tsx @@ -12,6 +12,7 @@ import styles from "./ConnectionsFilters.module.scss"; import { getAvailableDestinationOptions, getAvailableSourceOptions, statusFilterOptions } from "./filterOptions"; export interface FilterValues { + search: string; status: string | null; source: string | null; destination: string | null; @@ -89,11 +90,11 @@ export const ConnectionsFilters: React.FC = ({ { setFilters({ + search: "", status: null, source: null, destination: null, }); - setSearchFilter(""); }} /> diff --git a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageHeader.tsx b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageHeader.tsx index 9bf4ccd940c..e21c7d33454 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageHeader.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionPageHeader.tsx @@ -8,6 +8,7 @@ import { PageHeaderWithNavigation } from "components/ui/PageHeader"; import { Tabs, LinkTab } from "components/ui/Tabs"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; +import { useExperiment } from "hooks/services/Experiment"; import { RoutePaths, ConnectionRoutePaths } from "pages/routePaths"; import { ConnectionTitleBlock } from "./ConnectionTitleBlock"; @@ -17,6 +18,7 @@ export const ConnectionPageHeader = () => { const basePath = `/${RoutePaths.Workspaces}/${params.workspaceId}/${RoutePaths.Connections}/${params.connectionId}`; const { formatMessage } = useIntl(); const currentTab = params["*"] || ConnectionRoutePaths.Status; + const isSimplifiedCreation = useExperiment("connection.simplifiedCreation", false); const { connection, schemaRefreshing } = useConnectionEditService(); const breadcrumbsData = [ @@ -45,7 +47,7 @@ export const ConnectionPageHeader = () => { id: ConnectionRoutePaths.Replication, name: ( - + ), @@ -67,7 +69,7 @@ export const ConnectionPageHeader = () => { ]; return tabs; - }, [basePath, connection.schemaChange, schemaRefreshing]); + }, [basePath, connection.schemaChange, schemaRefreshing, isSimplifiedCreation]); return ( diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx index b765ef1e6f0..e9f88cdafeb 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx @@ -47,7 +47,7 @@ jest.mock("core/api", () => ({ useGetDestinationDefinitionSpecification: () => mockDestinationDefinitionSpecification, useSourceDefinition: () => mockSourceDefinition, useDestinationDefinition: () => mockDestinationDefinition, - LogsRequestError: jest.requireActual("core/api/errors").LogsRequestError, + ErrorWithJobInfo: jest.requireActual("core/api/errors").ErrorWithJobInfo, })); jest.mock("core/utils/rbac", () => ({ diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx index afaff514f91..4abe11b2617 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx @@ -154,19 +154,21 @@ export const ConnectionReplicationPage: React.FC = () => { return agg; }, {}); - const hasUserChangesInEnabledStreamsRequiringReset = values.syncCatalog.streams.some((_stream) => { - const formStream = structuredClone(_stream); - const connectionStream = structuredClone(lookupConnectionValuesStreamById[getStreamId(formStream)]); - - return !compareObjectsByFields(formStream.config, connectionStream.config, [ - "cursorField", - "destinationSyncMode", - "primaryKey", - "selectedFields", - "syncMode", - "aliasName", - ]); - }); + const hasUserChangesInEnabledStreamsRequiringReset = values.syncCatalog.streams + .filter((streamNode) => streamNode.config?.selected) + .some((streamNode) => { + const formStream = structuredClone(streamNode); + const connectionStream = structuredClone(lookupConnectionValuesStreamById[getStreamId(formStream)]); + + return !compareObjectsByFields(formStream.config, connectionStream.config, [ + "cursorField", + "destinationSyncMode", + "primaryKey", + "selectedFields", + "syncMode", + "aliasName", + ]); + }); const catalogChangesRequireReset = hasCatalogDiffInEnabledStream || hasUserChangesInEnabledStreamsRequiringReset; @@ -183,7 +185,7 @@ export const ConnectionReplicationPage: React.FC = () => { size: "md", content: (props) => , }); - if (result.type === "closed" && isBoolean(result.reason)) { + if (result.type === "completed" && isBoolean(result.reason)) { // Save the connection taking into account the correct skipReset value from the dialog choice. await saveConnection(values, !result.reason /* skipReset */); } else { diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx index 08880ac8519..95d8a41ce33 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx @@ -9,12 +9,12 @@ import { Text } from "components/ui/Text"; import { ConnectionStateType } from "core/api/types/AirbyteClient"; interface ResetWarningModalProps { - onClose: (withReset: boolean) => void; + onComplete: (withReset: boolean) => void; onCancel: () => void; stateType: ConnectionStateType; } -export const ResetWarningModal: React.FC = ({ onCancel, onClose, stateType }) => { +export const ResetWarningModal: React.FC = ({ onCancel, onComplete, stateType }) => { const { formatMessage } = useIntl(); const [withReset, setWithReset] = useState(true); const requireFullReset = stateType === ConnectionStateType.legacy; @@ -42,7 +42,7 @@ export const ResetWarningModal: React.FC = ({ onCancel, - diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap index 95ecc5b9b59..62de775d9ea 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap @@ -656,69 +656,13 @@ exports[`ConnectionReplicationPage should render 1`] = `
    -
    - -
    -
    -
    + />
    -
    -
    -

    - All -

    -
    -
    -
    + />
    diff --git a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx index eb69cc93fdb..63f2c982b01 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx @@ -5,7 +5,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; import { ConnectionDangerBlock } from "components/common/ConnectionDangerBlock"; -import { DeleteBlock } from "components/common/DeleteBlock"; +import { ConnectionDeleteBlock } from "components/common/ConnectionDeleteBlock"; import { FormConnectionFormValues, useConnectionValidationSchema, @@ -73,14 +73,12 @@ const dataResidencyDropdownDescription = ( export const ConnectionSettingsPage: React.FC = () => { const { connection, updateConnection } = useConnectionEditService(); const { mode } = useConnectionFormService(); - const { mutateAsync: deleteConnection } = useDeleteConnection(); const canUpdateDataResidency = useFeature(FeatureItem.AllowChangeDataGeographies); const canSendSchemaUpdateNotifications = useFeature(FeatureItem.AllowAutoDetectSchema); const { registerNotification } = useNotificationService(); const { formatMessage } = useIntl(); const { trackError } = useAppMonitoringService(); useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_SETTINGS); - const onDelete = () => deleteConnection(connection); const { workspaceId } = useCurrentWorkspace(); const canEditConnection = useIntent("EditConnection", { workspaceId }); @@ -165,7 +163,7 @@ export const ConnectionSettingsPage: React.FC = () => { - {connection.status !== "deprecated" && } + {connection.status !== "deprecated" && }     {({ open }) => ( @@ -257,7 +255,8 @@ const SimplifiedConnectionSettingsPage = () => { > diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamActionsMenu.module.scss b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamActionsMenu.module.scss new file mode 100644 index 00000000000..b238e892235 --- /dev/null +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamActionsMenu.module.scss @@ -0,0 +1,11 @@ +@use "scss/colors"; + +.streamActionsMenu { + &__clearDataModalStreamName { + font-style: italic; + } + + &__clearDataLabel > p { + color: colors.$red; + } +} diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamActionsMenu.tsx b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamActionsMenu.tsx index 1d6e0f07ecc..66f64fb51ae 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamActionsMenu.tsx +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamActionsMenu.tsx @@ -1,32 +1,44 @@ +import classNames from "classnames"; import React from "react"; -import { useIntl } from "react-intl"; +import { FormattedMessage, useIntl } from "react-intl"; import { useNavigate } from "react-router-dom"; import { useConnectionSyncContext } from "components/connection/ConnectionSync/ConnectionSyncContext"; import { StreamWithStatus } from "components/connection/StreamStatus/streamStatusUtils"; +import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; import { DropdownMenu, DropdownMenuOptionType } from "components/ui/DropdownMenu"; +import { Text } from "components/ui/Text"; +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; +import { useExperiment } from "hooks/services/Experiment"; import { ConnectionRoutePaths } from "pages/routePaths"; +import styles from "./StreamActionsMenu.module.scss"; + interface StreamActionsMenuProps { - streamState?: StreamWithStatus | undefined; + streamState: StreamWithStatus; } export const StreamActionsMenu: React.FC = ({ streamState }) => { const { formatMessage } = useIntl(); const navigate = useNavigate(); - + const sayClearInsteadOfReset = useExperiment("connection.clearNotReset", false); const { syncStarting, jobSyncRunning, resetStarting, jobResetRunning, resetStreams } = useConnectionSyncContext(); const { mode } = useConnectionFormService(); + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); const options: DropdownMenuOptionType[] = [ - { - displayName: formatMessage({ id: "connection.stream.actions.resetThisStream" }), - value: "resetThisStream", - disabled: syncStarting || jobSyncRunning || resetStarting || jobResetRunning || mode === "readonly", - }, + ...(sayClearInsteadOfReset + ? [] + : [ + { + displayName: formatMessage({ id: "connection.stream.actions.resetThisStream" }), + value: "resetThisStream", + disabled: syncStarting || jobSyncRunning || resetStarting || jobResetRunning || mode === "readonly", + }, + ]), { displayName: formatMessage({ id: "connection.stream.actions.showInReplicationTable" }), value: "showInReplicationTable", @@ -35,6 +47,18 @@ export const StreamActionsMenu: React.FC = ({ streamStat displayName: formatMessage({ id: "connection.stream.actions.openDetails" }), value: "openDetails", }, + ...(!sayClearInsteadOfReset + ? [] + : [ + { + displayName: formatMessage({ + id: "connection.stream.actions.clearData", + }), + value: "clearStreamData", + disabled: syncStarting || jobSyncRunning || resetStarting || jobResetRunning || mode === "readonly", + className: classNames(styles.streamActionsMenu__clearDataLabel), + }, + ]), ]; const onOptionClick = async ({ value }: DropdownMenuOptionType) => { @@ -44,6 +68,34 @@ export const StreamActionsMenu: React.FC = ({ streamStat }); } + if (value === "clearStreamData") { + openConfirmationModal({ + title: ( + {streamState.streamName} + ), + }} + /> + ), + text: "connection.stream.actions.clearData.confirm.text", + additionalContent: ( + + + + + + ), + submitButtonText: "connection.stream.actions.clearData.confirm.submit", + cancelButtonText: "connection.stream.actions.clearData.confirm.cancel", + onSubmit: async () => { + await resetStreams([{ streamNamespace: streamState.streamNamespace, streamName: streamState.streamName }]); + closeConfirmationModal(); + }, + }); + } if (value === "resetThisStream" && streamState) { await resetStreams([{ streamNamespace: streamState.streamNamespace, streamName: streamState.streamName }]); } diff --git a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderEditPage/ConnectorBuilderEditPage.tsx b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderEditPage/ConnectorBuilderEditPage.tsx index f5a043f2c58..701b4a829d2 100644 --- a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderEditPage/ConnectorBuilderEditPage.tsx +++ b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderEditPage/ConnectorBuilderEditPage.tsx @@ -9,7 +9,7 @@ import { HeadTitle } from "components/common/HeadTitle"; import { Builder } from "components/connectorBuilder/Builder/Builder"; import { StreamTestingPanel } from "components/connectorBuilder/StreamTestingPanel"; import { BuilderState, builderStateValidationSchema, useBuilderWatch } from "components/connectorBuilder/types"; -import { YamlEditor } from "components/connectorBuilder/YamlEditor"; +import { YamlManifestEditor } from "components/connectorBuilder/YamlEditor"; import { ResizablePanels } from "components/ui/ResizablePanels"; import { @@ -103,10 +103,16 @@ const Panels = React.memo(() => { panels={[ { children: ( - <>{mode === "yaml" ? : 1} />} + <> + {mode === "yaml" ? ( + + ) : ( + 1} /> + )} + ), className: styles.leftPanel, - minWidth: 550, + minWidth: 350, }, { children: , diff --git a/airbyte-webapp/src/pages/destination/AllDestinationsPage/AllDestinationsPage.tsx b/airbyte-webapp/src/pages/destination/AllDestinationsPage/AllDestinationsPage.tsx index 67803195a41..68e1e414962 100644 --- a/airbyte-webapp/src/pages/destination/AllDestinationsPage/AllDestinationsPage.tsx +++ b/airbyte-webapp/src/pages/destination/AllDestinationsPage/AllDestinationsPage.tsx @@ -1,4 +1,4 @@ -import React, { useDeferredValue, useMemo, useState } from "react"; +import React, { useDeferredValue, useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { Navigate, useNavigate } from "react-router-dom"; @@ -14,14 +14,15 @@ import { PageHeader } from "components/ui/PageHeader"; import { SearchInput } from "components/ui/SearchInput"; import { Text } from "components/ui/Text"; -import { useConnectionList, useCurrentWorkspace, useDestinationList } from "core/api"; +import { useConnectionList, useCurrentWorkspace, useDestinationList, useFilters } from "core/api"; +import { DestinationRead } from "core/api/types/AirbyteClient"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { useIntent } from "core/utils/rbac"; import styles from "./AllDestinationsPage.module.scss"; import { DestinationPaths } from "../../routePaths"; -export const AllDestinationsPage: React.FC = () => { +const AllDestinationsPageInner: React.FC<{ destinations: DestinationRead[] }> = ({ destinations }) => { const navigate = useNavigate(); useTrackPage(PageTrackingCodes.DESTINATION_LIST); @@ -29,13 +30,12 @@ export const AllDestinationsPage: React.FC = () => { const { workspaceId } = useCurrentWorkspace(); const canCreateDestination = useIntent("CreateDestination", { workspaceId }); - const { destinations } = useDestinationList(); const connectionList = useConnectionList({ destinationId: destinations.map(({ destinationId }) => destinationId) }); const connections = connectionList?.connections ?? []; const data = getEntityTableData(destinations, connections, "destination"); - const [searchFilter, setSearchFilter] = useState(""); - const debouncedSearchFilter = useDeferredValue(searchFilter); + const [{ search }, setFilterValue] = useFilters<{ search: string }>({ search: "" }); + const debouncedSearchFilter = useDeferredValue(search); const filteredDestinations = useMemo( () => filterBySearchEntityTableData(debouncedSearchFilter, data), @@ -69,7 +69,7 @@ export const AllDestinationsPage: React.FC = () => { > - setSearchFilter(value)} /> + setFilterValue("search", value)} /> {filteredDestinations.length === 0 && ( @@ -85,3 +85,12 @@ export const AllDestinationsPage: React.FC = () => { ); }; + +export const AllDestinationsPage: React.FC = () => { + const { destinations } = useDestinationList(); + return destinations.length ? ( + + ) : ( + + ); +}; diff --git a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx index 40af86fa289..0334454692f 100644 --- a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx @@ -77,7 +77,7 @@ export const DestinationSettingsPage: React.FC = () => { ); }, [connectionsWithDestination]); - const onDeleteClick = useDeleteModal("destination", onDelete, modalAdditionalContent); + const onDeleteClick = useDeleteModal("destination", onDelete, modalAdditionalContent, destination.name); return (
    diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index 39589d8d3cb..81f5537e481 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -49,7 +49,7 @@ const SourceSettingsPage = React.lazy(() => import("./source/SourceSettingsPage" const SourceConnectionsPage = React.lazy(() => import("./source/SourceConnectionsPage")); const AdvancedSettingsPage = React.lazy(() => import("./SettingsPage/pages/AdvancedSettingsPage")); -const WorkspacesPage = React.lazy(() => import("./workspaces/WorkspacesPage")); +const WorkspacesPage = React.lazy(() => import("./workspaces")); const useAddAnalyticsContextForWorkspace = (workspace: WorkspaceRead): void => { const analyticsContext = useMemo( diff --git a/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx b/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx index dc9d03d69df..5e12e434e84 100644 --- a/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx +++ b/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx @@ -1,4 +1,4 @@ -import React, { useDeferredValue, useMemo, useState } from "react"; +import React, { useDeferredValue, useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { Navigate, useNavigate } from "react-router-dom"; @@ -14,26 +14,26 @@ import { PageHeader } from "components/ui/PageHeader"; import { SearchInput } from "components/ui/SearchInput"; import { Text } from "components/ui/Text"; -import { useConnectionList, useCurrentWorkspace, useSourceList } from "core/api"; +import { useConnectionList, useCurrentWorkspace, useFilters, useSourceList } from "core/api"; +import { SourceRead } from "core/api/types/AirbyteClient"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { useIntent } from "core/utils/rbac"; import styles from "./AllSourcesPage.module.scss"; import { SourcePaths } from "../../routePaths"; -const AllSourcesPage: React.FC = () => { +const AllSourcesPageInner: React.FC<{ sources: SourceRead[] }> = ({ sources }) => { const navigate = useNavigate(); useTrackPage(PageTrackingCodes.SOURCE_LIST); const onCreateSource = () => navigate(`${SourcePaths.SelectSourceNew}`); const { workspaceId } = useCurrentWorkspace(); const canCreateSource = useIntent("CreateSource", { workspaceId }); - const { sources } = useSourceList(); const connectionList = useConnectionList({ sourceId: sources.map(({ sourceId }) => sourceId) }); const connections = connectionList?.connections ?? []; const data = getEntityTableData(sources, connections, "source"); - const [searchFilter, setSearchFilter] = useState(""); - const debouncedSearchFilter = useDeferredValue(searchFilter); + const [{ search }, setFilterValue] = useFilters<{ search: string }>({ search: "" }); + const debouncedSearchFilter = useDeferredValue(search); const filteredSources = useMemo( () => filterBySearchEntityTableData(debouncedSearchFilter, data), @@ -61,7 +61,7 @@ const AllSourcesPage: React.FC = () => { > - setSearchFilter(value)} /> + setFilterValue("search", value)} /> {filteredSources.length === 0 && ( @@ -78,4 +78,9 @@ const AllSourcesPage: React.FC = () => { ); }; +const AllSourcesPage: React.FC = () => { + const { sources } = useSourceList(); + return sources.length ? : ; +}; + export default AllSourcesPage; diff --git a/airbyte-webapp/src/pages/source/CreateSourcePage/SourceForm.tsx b/airbyte-webapp/src/pages/source/CreateSourcePage/SourceForm.tsx index d06fbf7b829..c17bd6ad737 100644 --- a/airbyte-webapp/src/pages/source/CreateSourcePage/SourceForm.tsx +++ b/airbyte-webapp/src/pages/source/CreateSourcePage/SourceForm.tsx @@ -7,10 +7,9 @@ import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { ConnectionConfiguration } from "area/connector/types"; -import { useGetSourceDefinitionSpecificationAsync, LogsRequestError } from "core/api"; +import { useGetSourceDefinitionSpecificationAsync } from "core/api"; import { SourceDefinitionRead } from "core/api/types/AirbyteClient"; import { Connector } from "core/domain/connector"; -import { FormError } from "core/utils/errorStatusMessage"; import { ConnectorCard } from "views/Connector/ConnectorCard"; import { ConnectorCardValues } from "views/Connector/ConnectorForm/types"; @@ -24,7 +23,6 @@ export interface SourceFormValues { interface SourceFormProps { onSubmit: (values: SourceFormValues) => Promise; sourceDefinitions: SourceDefinitionRead[]; - error?: FormError | null; selectedSourceDefinitionId?: string; } @@ -36,12 +34,7 @@ const hasSourceDefinitionId = (state: unknown): state is { sourceDefinitionId: s ); }; -export const SourceForm: React.FC = ({ - onSubmit, - sourceDefinitions, - error, - selectedSourceDefinitionId, -}) => { +export const SourceForm: React.FC = ({ onSubmit, sourceDefinitions, selectedSourceDefinitionId }) => { const location = useLocation(); const [sourceDefinitionId, setSourceDefinitionId] = useState( @@ -92,7 +85,6 @@ export const SourceForm: React.FC = ({ selectedConnectorDefinitionSpecification={sourceDefinitionSpecification} selectedConnectorDefinitionId={sourceDefinitionId} onSubmit={onSubmitForm} - jobInfo={LogsRequestError.extractJobInfo(error)} supportLevel={selectedSourceDefinition?.supportLevel} /> ); diff --git a/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx b/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx index d707d965f52..01acd978386 100644 --- a/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx +++ b/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx @@ -74,7 +74,7 @@ export const SourceSettingsPage: React.FC = () => { ); }, [connectionsWithSource]); - const onDeleteClick = useDeleteModal("source", onDelete, modalAdditionalContent); + const onDeleteClick = useDeleteModal("source", onDelete, modalAdditionalContent, source.name); return (
    diff --git a/airbyte-webapp/src/pages/workspaces/WorkspacesPage.tsx b/airbyte-webapp/src/pages/workspaces/WorkspacesPage.tsx index 13bf4a8a3e0..b91382704c6 100644 --- a/airbyte-webapp/src/pages/workspaces/WorkspacesPage.tsx +++ b/airbyte-webapp/src/pages/workspaces/WorkspacesPage.tsx @@ -26,7 +26,7 @@ import styles from "./WorkspacesPage.module.scss"; export const WORKSPACE_LIST_LENGTH = 50; -const WorkspacesPage: React.FC = () => { +export const WorkspacesPage: React.FC = () => { const { isLoading: isLogoutLoading, mutateAsync: handleLogout } = useMutation(() => logout?.() ?? Promise.resolve()); useTrackPage(PageTrackingCodes.WORKSPACES); const [searchValue, setSearchValue] = useState(""); @@ -120,5 +120,3 @@ const WorkspacesPage: React.FC = () => { ); }; - -export default WorkspacesPage; diff --git a/airbyte-webapp/src/pages/workspaces/index.tsx b/airbyte-webapp/src/pages/workspaces/index.tsx index 78fe36a1baf..979270bc9b0 100644 --- a/airbyte-webapp/src/pages/workspaces/index.tsx +++ b/airbyte-webapp/src/pages/workspaces/index.tsx @@ -1,3 +1 @@ -import WorkspacesPage from "./WorkspacesPage"; - -export default WorkspacesPage; +export { WorkspacesPage as default } from "./WorkspacesPage"; diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index ff2d426a10a..6d1088cc275 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -2,6 +2,7 @@ import { UseMutateAsyncFunction, UseQueryResult } from "@tanstack/react-query"; import { dump } from "js-yaml"; import cloneDeep from "lodash/cloneDeep"; import isEqual from "lodash/isEqual"; +import toPath from "lodash/toPath"; import React, { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; import { useFormContext, UseFormReturn } from "react-hook-form"; import { useIntl } from "react-intl"; @@ -12,9 +13,10 @@ import { WaitForSavingModal } from "components/connectorBuilder/Builder/WaitForS import { convertToBuilderFormValuesSync } from "components/connectorBuilder/convertManifestToBuilderForm"; import { BuilderState, + convertToManifest, DEFAULT_BUILDER_FORM_VALUES, DEFAULT_JSON_MANIFEST_VALUES, - convertToManifest, + getManifestValuePerComponentPerStream, useBuilderWatch, } from "components/connectorBuilder/types"; import { useManifestToBuilderForm } from "components/connectorBuilder/useManifestToBuilderForm"; @@ -28,14 +30,14 @@ import { CommonRequestError, NewVersionBody, useBuilderProject, - usePublishBuilderProject, - useReleaseNewBuilderProjectVersion, - useUpdateBuilderProject, + useBuilderProjectReadStream, + useBuilderProjectUpdateTestingValues, useBuilderResolvedManifest, useBuilderResolvedManifestSuspense, useCurrentWorkspace, - useBuilderProjectReadStream, - useBuilderProjectUpdateTestingValues, + usePublishBuilderProject, + useReleaseNewBuilderProjectVersion, + useUpdateBuilderProject, } from "core/api"; import { useIsForeignWorkspace } from "core/api/cloud"; import { @@ -79,6 +81,11 @@ interface FormStateContext { previousManifestDraft: DeclarativeComponentSchema | undefined; displayedVersion: number | undefined; formValuesValid: boolean; + resolvedManifest: ConnectorManifest; + resolveErrorMessage: string | undefined; + resolveError: CommonRequestError | null; + isResolving: boolean; + streamNames: string[]; setDisplayedVersion: (value: number | undefined, manifest: DeclarativeComponentSchema) => void; updateJsonManifest: (jsonValue: ConnectorManifest) => void; setYamlIsValid: (value: boolean) => void; @@ -103,9 +110,6 @@ interface TestReadLimits { } export interface TestReadContext { - resolvedManifest: ConnectorManifest; - resolveErrorMessage: string | undefined; - resolveError: CommonRequestError | null; streamRead: UseQueryResult; testReadLimits: { recordLimit: number; @@ -116,7 +120,6 @@ export interface TestReadContext { setSliceLimit: (newSliceLimit: number) => void; defaultLimits: TestReadLimits; }; - isResolving: boolean; schemaWarnings: { schemaDifferences: boolean; incompatibleSchemaErrors: string[] | undefined; @@ -128,7 +131,7 @@ interface FormManagementStateContext { setTestingValuesInputOpen: (open: boolean) => void; isTestReadSettingsOpen: boolean; setTestReadSettingsOpen: (open: boolean) => void; - scrollToField: string | undefined; + handleScrollToField: (ref: React.RefObject, path: string) => void; setScrollToField: (field: string | undefined) => void; stateKey: number; setStateKey: React.Dispatch>; @@ -157,15 +160,45 @@ export const ConnectorBuilderFormStateProvider: React.FC = [ + "version", + "type", + "check", + "definitions", + "streams", + "spec", + "metadata", + "schemas", +]; +export function convertJsonToYaml(json: ConnectorManifest): string { + const yamlString = dump(json, { noRefs: true, + sortKeys: (a: keyof ConnectorManifest, b: keyof ConnectorManifest) => { + const orderA = MANIFEST_KEY_ORDER.indexOf(a); + const orderB = MANIFEST_KEY_ORDER.indexOf(b); + if (orderA === -1 && orderB === -1) { + return 0; + } + if (orderA === -1) { + return 1; + } + if (orderB === -1) { + return -1; + } + return orderA - orderB; + }, + }); + + // add newlines between root-level fields + return yamlString.replace(/^\S+.*/gm, (match, offset) => { + return offset > 0 ? `\n${match}` : match; }); } export const InternalConnectorBuilderFormStateProvider: React.FC< React.PropsWithChildren<{ permission: ConnectorBuilderPermission }> > = ({ children, permission }) => { + const { formatMessage } = useIntl(); const { projectId, builderProject, updateProject, updateError } = useInitializedBuilderProject(); const currentProject: BuilderProject = useMemo( @@ -198,10 +231,51 @@ export const InternalConnectorBuilderFormStateProvider: React.FC< const [yamlEditorIsMounted, setYamlEditorIsMounted] = useState(true); const [formValuesValid, setFormValuesValid] = useState(true); + const workspaceId = useCurrentWorkspaceId(); + const { setValue, getValues } = useFormContext(); const mode = useBuilderWatch("mode"); const name = useBuilderWatch("name"); + const manifestValuePerComponentPerStream = useMemo( + () => (mode === "ui" ? getManifestValuePerComponentPerStream(jsonManifest) : undefined), + [jsonManifest, mode] + ); + + const { + data: resolveData, + isError: isResolveError, + error: resolveError, + isFetching: isResolving, + } = useBuilderResolvedManifest( + { + manifest: jsonManifest, + workspace_id: workspaceId, + project_id: projectId, + form_generated_manifest: mode === "ui", + }, + // In UI mode, we only need to call resolve if we have YAML components + mode === "yaml" || (mode === "ui" && !!jsonManifest.metadata?.yamlComponents), + manifestValuePerComponentPerStream + ); + const unknownErrorMessage = formatMessage({ id: "connectorBuilder.unknownError" }); + const resolveErrorMessage = isResolveError + ? resolveError instanceof Error + ? resolveError.message || unknownErrorMessage + : unknownErrorMessage + : undefined; + + // In UI mode, we can treat the jsonManifest as resolved, since the resolve call is only used to check for invalid YAML + // components in that case. + // Using the resolve data manifest as the resolved manifest would introduce an unnecessary lag effect in UI mode, where + // test reads would use the old manifest until the resolve call completes. + const resolvedManifest = + mode === "ui" ? jsonManifest : ((resolveData?.manifest ?? DEFAULT_JSON_MANIFEST_VALUES) as ConnectorManifest); + + const streams = useBuilderWatch("formValues.streams"); + const streamNames = + mode === "ui" ? streams.map((stream) => stream.name) : resolvedManifest.streams.map((stream) => stream.name ?? ""); + useEffect(() => { if (name !== currentProject.name) { setPreviousManifestDraft(undefined); @@ -229,12 +303,7 @@ export const InternalConnectorBuilderFormStateProvider: React.FC< const toggleUI = useCallback( async (newMode: BuilderState["mode"]) => { if (newMode === "yaml") { - setValue( - "yaml", - dump(jsonManifest, { - noRefs: true, - }) - ); + setValue("yaml", convertJsonToYaml(jsonManifest)); setYamlIsValid(true); setValue("mode", "yaml"); } else { @@ -353,10 +422,12 @@ export const InternalConnectorBuilderFormStateProvider: React.FC< [sendNewVersionRequest] ); + const formAndResolveValid = useMemo(() => formValuesValid && resolveError === null, [formValuesValid, resolveError]); + const savingState = getSavingState( jsonManifest, yamlIsValid, - formValuesValid, + formAndResolveValid, mode, name, persistedState, @@ -377,13 +448,13 @@ export const InternalConnectorBuilderFormStateProvider: React.FC< return; } // do not save invalid ui-based manifest (e.g. no streams), but always save yaml-based manifest - if (modeRef.current === "ui" && !formValuesValid) { + if (modeRef.current === "ui" && !formAndResolveValid) { return; } const newProject: BuilderProjectWithManifest = { name, manifest: jsonManifest }; await updateProject(newProject); setPersistedState(newProject); - }, [permission, name, formValuesValid, jsonManifest, updateProject]); + }, [permission, name, formAndResolveValid, jsonManifest, updateProject]); useDebounce( () => { @@ -421,6 +492,11 @@ export const InternalConnectorBuilderFormStateProvider: React.FC< previousManifestDraft, displayedVersion, formValuesValid, + resolvedManifest, + resolveError, + resolveErrorMessage, + isResolving, + streamNames, setDisplayedVersion: setToVersion, updateJsonManifest, setYamlIsValid, @@ -492,23 +568,21 @@ export function useInitializedBuilderProject() { } const builderProject = useBuilderProject(projectId); const { mutateAsync: updateProject, error: updateError } = useUpdateBuilderProject(projectId); + const persistedManifest = + (builderProject.declarativeManifest?.manifest as ConnectorManifest) ?? DEFAULT_JSON_MANIFEST_VALUES; const resolvedManifest = useBuilderResolvedManifestSuspense(builderProject.declarativeManifest?.manifest, projectId); const [initialFormValues, failedInitialFormValueConversion, initialYaml] = useMemo(() => { if (!resolvedManifest) { // could not resolve manifest, use default form values - return [ - DEFAULT_BUILDER_FORM_VALUES, - true, - convertJsonToYaml(builderProject.declarativeManifest?.manifest ?? DEFAULT_JSON_MANIFEST_VALUES), - ]; + return [DEFAULT_BUILDER_FORM_VALUES, true, convertJsonToYaml(persistedManifest)]; } try { - return [convertToBuilderFormValuesSync(resolvedManifest), false, convertJsonToYaml(resolvedManifest)]; + return [convertToBuilderFormValuesSync(resolvedManifest), false, convertJsonToYaml(persistedManifest)]; } catch (e) { // could not convert to form values, use default form values - return [DEFAULT_BUILDER_FORM_VALUES, true, convertJsonToYaml(resolvedManifest)]; + return [DEFAULT_BUILDER_FORM_VALUES, true, convertJsonToYaml(persistedManifest)]; } - }, [builderProject.declarativeManifest?.manifest, resolvedManifest]); + }, [persistedManifest, resolvedManifest]); return { projectId, @@ -537,9 +611,7 @@ function useBlockOnSavingState(savingState: SavingState) { closeConfirmationModal(); blocker.proceed(); }, - onClose: () => { - setBlockedOnInvalidState(false); - }, + onCancel: () => setBlockedOnInvalidState(false), }); } else { setPendingBlocker(blocker); @@ -562,7 +634,7 @@ function useBlockOnSavingState(savingState: SavingState) { function getSavingState( currentJsonManifest: ConnectorManifest, yamlIsValid: boolean, - formValuesValid: boolean, + formAndResolveValid: boolean, mode: BuilderState["mode"], name: string | undefined, persistedState: { name: string; manifest?: DeclarativeComponentSchema }, @@ -576,7 +648,7 @@ function getSavingState( if (name === undefined) { return "invalid"; } - if (mode === "ui" && !formValuesValid) { + if (mode === "ui" && !formAndResolveValid) { return "invalid"; } if (mode === "yaml" && !yamlIsValid) { @@ -595,37 +667,13 @@ function getSavingState( } export const ConnectorBuilderTestReadProvider: React.FC> = ({ children }) => { - const { formatMessage } = useIntl(); const workspaceId = useCurrentWorkspaceId(); - const { jsonManifest, projectId } = useConnectorBuilderFormState(); + const { projectId, resolvedManifest } = useConnectorBuilderFormState(); const { setValue } = useFormContext(); const mode = useBuilderWatch("mode"); const view = useBuilderWatch("view"); const testStreamIndex = useBuilderWatch("testStreamIndex"); - - const manifest = jsonManifest ?? DEFAULT_JSON_MANIFEST_VALUES; - - const { - data, - isError: isResolveError, - error: resolveError, - isFetching: isResolving, - } = useBuilderResolvedManifest( - { - manifest, - workspace_id: workspaceId, - project_id: projectId, - form_generated_manifest: mode === "ui", - }, - // don't need to resolve manifest in UI mode since it doesn't use $refs or $parameters - mode === "yaml" - ); - const unknownErrorMessage = formatMessage({ id: "connectorBuilder.unknownError" }); - const resolveErrorMessage = isResolveError - ? resolveError instanceof Error - ? resolveError.message || unknownErrorMessage - : unknownErrorMessage - : undefined; + const streams = useBuilderWatch("formValues.streams"); useEffect(() => { if (typeof view === "number") { @@ -633,14 +681,12 @@ export const ConnectorBuilderTestReadProvider: React.FC { export const useSelectedPageAndSlice = () => { const { resolvedManifest: { streams }, - } = useConnectorBuilderTestRead(); + } = useConnectorBuilderFormState(); const testStreamIndex = useBuilderWatch("testStreamIndex"); const selectedStreamName = streams[testStreamIndex]?.name ?? ""; @@ -795,6 +837,11 @@ export const useSelectedPageAndSlice = () => { return { selectedSlice, selectedPage, setSelectedSlice, setSelectedPage }; }; +// check whether paths are equal, normalizing [] and . notation +function arePathsEqual(path1: string, path2: string) { + return isEqual(toPath(path1), toPath(path2)); +} + export const ConnectorBuilderFormManagementStateProvider: React.FC> = ({ children, }) => { @@ -803,18 +850,28 @@ export const ConnectorBuilderFormManagementStateProvider: React.FC(undefined); const [stateKey, setStateKey] = useState(0); + const handleScrollToField = useCallback( + (ref: React.RefObject, path: string) => { + if (ref.current && scrollToField && arePathsEqual(path, scrollToField)) { + ref.current.scrollIntoView({ block: "center" }); + setScrollToField(undefined); + } + }, + [scrollToField] + ); + const ctx = useMemo( () => ({ isTestingValuesInputOpen, setTestingValuesInputOpen, isTestReadSettingsOpen, setTestReadSettingsOpen, - scrollToField, + handleScrollToField, setScrollToField, stateKey, setStateKey, }), - [isTestingValuesInputOpen, isTestReadSettingsOpen, scrollToField, stateKey] + [isTestingValuesInputOpen, isTestReadSettingsOpen, handleScrollToField, stateKey] ); return ( diff --git a/airbyte-webapp/src/test-utils/setup-tests.ts b/airbyte-webapp/src/test-utils/setup-tests.ts index e846df72298..4f3ba839be1 100644 --- a/airbyte-webapp/src/test-utils/setup-tests.ts +++ b/airbyte-webapp/src/test-utils/setup-tests.ts @@ -22,3 +22,8 @@ global.ResizeObserver = jest.fn().mockImplementation(() => ({ unobserve: jest.fn(), disconnect: jest.fn(), })); + +// retry failed tests when configured to (e.g. `test:ci`) +if (process.env.JEST_RETRIES) { + jest.retryTimes(parseInt(process.env.JEST_RETRIES, 10)); +} diff --git a/airbyte-webapp/src/test-utils/testutils.tsx b/airbyte-webapp/src/test-utils/testutils.tsx index 9423523021f..d318f8e9b60 100644 --- a/airbyte-webapp/src/test-utils/testutils.tsx +++ b/airbyte-webapp/src/test-utils/testutils.tsx @@ -11,7 +11,6 @@ import { SourceRead, WebBackendConnectionRead, } from "core/api/types/AirbyteClient"; -import { ConfigContext, config } from "core/config"; import { defaultOssFeatures, FeatureItem, FeatureService } from "core/services/features"; import { ConfirmationModalService } from "hooks/services/ConfirmationModal"; import { ModalServiceProvider } from "hooks/services/Modal"; @@ -51,19 +50,17 @@ export const TestWrapper: React.FC> features = defaultOssFeatures, }) => ( null}> - - - - - - - {children} - - - - - - + + + + + + {children} + + + + + ); diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx index b90641ce1fd..9976e49ee2c 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx @@ -12,8 +12,8 @@ import { Pre } from "components/ui/Pre"; import { Spinner } from "components/ui/Spinner"; import { useAirbyteCloudIps } from "area/connector/utils/useAirbyteCloudIps"; -import { LogsRequestError } from "core/api"; -import { DestinationRead, SourceRead, SupportLevel, SynchronousJobRead } from "core/api/types/AirbyteClient"; +import { ErrorWithJobInfo } from "core/api"; +import { DestinationRead, SourceRead, SupportLevel } from "core/api/types/AirbyteClient"; import { Connector, ConnectorDefinition, @@ -42,7 +42,6 @@ interface ConnectorCardBaseProps { headerBlock?: React.ReactNode; description?: React.ReactNode; full?: boolean; - jobInfo?: SynchronousJobRead | null; onSubmit: (values: ConnectorCardValues) => Promise | void; reloadConfig?: () => void; onDeleteClick?: () => void; @@ -79,7 +78,6 @@ const getConnectorId = (connectorRead: DestinationRead | SourceRead) => { }; export const ConnectorCard: React.FC = ({ - jobInfo, onSubmit, onDeleteClick, selectedConnectorDefinitionId, @@ -142,7 +140,7 @@ export const ConnectorCard: React.FC { resetConnectorForm(); }} diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestingConnectionSpinner.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestingConnectionSpinner.tsx deleted file mode 100644 index d1811a7aba8..00000000000 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestingConnectionSpinner.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import React from "react"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/ui/Button"; -import { ProgressBar } from "components/ui/ProgressBar"; - -import styles from "./TestingConnectionSpinner.module.scss"; - -// Progress Bar runs 2min for checking connections -const PROGRESS_BAR_TIME = 60 * 2; - -interface TestingConnectionSpinnerProps { - isCancellable?: boolean; - onCancelTesting?: () => void; -} - -export const TestingConnectionSpinner: React.FC = ({ - isCancellable, - onCancelTesting, -}) => ( -
    - - {isCancellable && ( - - )} -
    -); diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/ResourceNotAvailable.module.scss b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/ResourceNotAvailable.module.scss deleted file mode 100644 index 0fce061b21e..00000000000 --- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/ResourceNotAvailable.module.scss +++ /dev/null @@ -1,32 +0,0 @@ -@use "scss/variables"; -@use "scss/colors"; - -.requestContainer { - height: 91vh; - margin-top: variables.$spacing-xl; - width: 100%; - - &__erd--previewImage { - background-size: cover; - background-repeat: no-repeat; - background-image: linear-gradient(rgba(0, 0, 0, 50%), rgba(0, 0, 0, 50%)), url("./erd.png"); - } - - &__schema--previewImage { - background-size: cover; - background-repeat: no-repeat; - background-image: linear-gradient(rgba(0, 0, 0, 30%), rgba(0, 0, 0, 30%)), url("./schema.png"); - } - - &__messageBox { - padding: variables.$spacing-xl; - max-width: 70%; - background-color: colors.$blue-100; - border-radius: variables.$border-radius-md; - - p { - color: colors.$dark-blue-900; - text-align: center; - } - } -} diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/ResourceNotAvailable.tsx b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/ResourceNotAvailable.tsx deleted file mode 100644 index 48cc1b4b239..00000000000 --- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/ResourceNotAvailable.tsx +++ /dev/null @@ -1,74 +0,0 @@ -import classNames from "classnames"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/ui/Button"; -import { FlexContainer } from "components/ui/Flex"; -import { Text } from "components/ui/Text"; - -import { isSourceDefinition } from "core/domain/connector/source"; -import { useDocumentationPanelContext } from "views/Connector/ConnectorDocumentationLayout/DocumentationPanelContext"; - -import styles from "./ResourceNotAvailable.module.scss"; -import { useAnalyticsTrackFunctions } from "./useAnalyticsTrackFunctions"; - -interface ResourceNotAvailableProps { - activeTab: "erd" | "schema"; - isRequested: boolean; - setRequested: (val: boolean) => void; -} -export const ResourceNotAvailable: React.FC> = ({ - activeTab, - setRequested, - isRequested, -}) => { - const { selectedConnectorDefinition } = useDocumentationPanelContext(); - const { trackRequest } = useAnalyticsTrackFunctions(); - - return ( - - {isRequested ? ( -
    - - - -
    - ) : ( - - - - - - - - )} -     - ); -}; diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/erd.png b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/erd.png deleted file mode 100644 index 324000a9c54..00000000000 Binary files a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/erd.png and /dev/null differ diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/index.tsx b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/index.tsx deleted file mode 100644 index 7b089711a4f..00000000000 --- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export { ResourceNotAvailable } from "./ResourceNotAvailable"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/schema.png b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/schema.png deleted file mode 100644 index 0b23b279eec..00000000000 Binary files a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/schema.png and /dev/null differ diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/useAnalyticsTrackFunctions.tsx b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/useAnalyticsTrackFunctions.tsx deleted file mode 100644 index 922df8f6344..00000000000 --- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ResourceNotAvailable/useAnalyticsTrackFunctions.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { useCallback } from "react"; - -import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; - -export const useAnalyticsTrackFunctions = () => { - const analytics = useAnalyticsService(); - - const trackRequest = useCallback( - ({ - sourceDefinitionId, - connectorName, - requestType, - }: { - sourceDefinitionId: string; - connectorName: string; - requestType: "schema" | "erd"; - }) => { - const namespace = requestType === "schema" ? Namespace.SCHEMA : Namespace.ERD; - - analytics.track(namespace, Action.REQUEST, { - actionDescription: `Requested source ${requestType}`, - connector_source: connectorName, - connector_source_definition_id: sourceDefinitionId, - request_type: requestType, - }); - }, - [analytics] - ); - return { trackRequest }; -}; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx index 1e0d2a6c550..19d99710d1c 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx @@ -1,4 +1,5 @@ import classNames from "classnames"; +import { JSONSchema7Type } from "json-schema"; import pick from "lodash/pick"; import React, { useCallback, useMemo } from "react"; import { get, useFormContext, useFormState, useWatch } from "react-hook-form"; @@ -10,7 +11,7 @@ import { RadioButton } from "components/ui/RadioButton"; import { Text } from "components/ui/Text"; import { TextWithHTML } from "components/ui/TextWithHTML"; -import { FormConditionItem } from "core/form/types"; +import { FormConditionItem, FormGroupItem } from "core/form/types"; import { useOptionalDocumentationPanelContext } from "views/Connector/ConnectorDocumentationLayout/DocumentationPanelContext"; import styles from "./ConditionSection.module.scss"; @@ -33,7 +34,10 @@ export const ConditionSection: React.FC = ({ formField, p const setFocusedField = useOptionalDocumentationPanelContext()?.setFocusedField; const value = useWatch({ name: path }); - const { conditions, selectionConstValues } = formField; + const { conditions, selectionConstValues } = useMemo( + () => getVisibleConditionsAndConstValues(formField), + [formField] + ); const currentSelectionValue = useWatch({ name: `${path}.${formField.selectionKey}` }); let currentlySelectedCondition: number | undefined = selectionConstValues.indexOf(currentSelectionValue); if (currentlySelectedCondition === -1) { @@ -132,3 +136,22 @@ export const ConditionSection: React.FC = ({ formField, p ); }; + +const getVisibleConditionsAndConstValues = ( + formField: FormConditionItem +): { conditions: FormGroupItem[]; selectionConstValues: JSONSchema7Type[] } => { + const conditions: FormGroupItem[] = []; + const selectionConstValues: JSONSchema7Type[] = []; + + formField.conditions.forEach((condition, index) => { + if (!condition.airbyte_hidden) { + conditions.push(condition); + selectionConstValues.push(formField.selectionConstValues[index]); + } + }); + + return { + conditions, + selectionConstValues, + }; +}; diff --git a/airbyte-webapp/src/views/Connector/RequestConnectorModal/index.tsx b/airbyte-webapp/src/views/Connector/RequestConnectorModal/index.tsx deleted file mode 100644 index 8e5195cc62e..00000000000 --- a/airbyte-webapp/src/views/Connector/RequestConnectorModal/index.tsx +++ /dev/null @@ -1,3 +0,0 @@ -import RequestConnectorModal from "./RequestConnectorModal"; - -export default RequestConnectorModal; diff --git a/airbyte-webapp/src/views/Connector/RequestConnectorModal/types.ts b/airbyte-webapp/src/views/Connector/RequestConnectorModal/types.ts deleted file mode 100644 index ffefb968849..00000000000 --- a/airbyte-webapp/src/views/Connector/RequestConnectorModal/types.ts +++ /dev/null @@ -1,6 +0,0 @@ -export interface Values { - connectorType: string; - name: string; - additionalInfo?: string; - email?: string; -} diff --git a/airbyte-webapp/src/views/layout/SideBar/MainNavItems.module.scss b/airbyte-webapp/src/views/layout/SideBar/MainNavItems.module.scss deleted file mode 100644 index d562b0ec6b8..00000000000 --- a/airbyte-webapp/src/views/layout/SideBar/MainNavItems.module.scss +++ /dev/null @@ -1,30 +0,0 @@ -@use "scss/colors"; -@use "scss/variables"; - -.beta { - overflow: hidden; - - &::before { - content: "beta"; - position: absolute; - top: -3px; - left: -19px; - transform: rotate(-45deg); - background-color: colors.$blue-400; - color: white; - padding: 14px 17px 3px; - opacity: 0; - transition: all variables.$transition-out; - font-size: 9px; - text-transform: uppercase; - font-weight: bold; - } - - &:hover::before { - opacity: 1; - } - - &--active::before { - opacity: 1; - } -} diff --git a/airbyte-webapp/src/views/layout/SideBar/MainNavItems.tsx b/airbyte-webapp/src/views/layout/SideBar/MainNavItems.tsx deleted file mode 100644 index 444ad0d21b3..00000000000 --- a/airbyte-webapp/src/views/layout/SideBar/MainNavItems.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import { FormattedMessage } from "react-intl"; - -import { RoutePaths } from "pages/routePaths"; - -import { MenuContent } from "./components/MenuContent"; -import { NavItem } from "./components/NavItem"; -import styles from "./MainNavItems.module.scss"; - -export const MainNavItems: React.FC = () => { - return ( - - } - icon="connection" - to={RoutePaths.Connections} - testId="connectionsLink" - /> - - } - icon="source" - to={RoutePaths.Source} - testId="sourcesLink" - /> - - } - icon="destination" - testId="destinationsLink" - to={RoutePaths.Destination} - /> - } - icon="wrench" - testId="builderLink" - to={RoutePaths.ConnectorBuilder} - className={styles.beta} - activeClassName={styles["beta--active"]} - /> - - ); -}; diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 532915fd8c9..75bd00f03b2 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -1,14 +1,18 @@ -FROM airbyte/airbyte-base-java-worker-image:2.0.1 +ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.1.0 +FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION} ENV APPLICATION airbyte-workers ENV VERSION ${VERSION} WORKDIR /app +USER root COPY WellKnownTypes.json /app # Move worker app ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte EXPOSE 5005 diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java index 1e303b10b52..9a416ee72ba 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java @@ -6,6 +6,8 @@ import io.airbyte.commons.temporal.TemporalConstants; import io.airbyte.commons.temporal.config.WorkerMode; +import io.airbyte.commons.temporal.utils.PayloadChecker; +import io.airbyte.metrics.lib.MetricClient; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.temporal.check.connection.CheckConnectionActivity; import io.airbyte.workers.temporal.discover.catalog.DiscoverCatalogActivity; @@ -88,6 +90,11 @@ public List connectionManagerActivities( appendToAttemptLogActivity); } + @Singleton + public PayloadChecker payloadChecker(final MetricClient metricClient) { + return new PayloadChecker(metricClient); + } + @Singleton @Named("discoverActivities") public List discoverActivities( diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionWorkflowImpl.java index e951e0d8ac9..3c0ee3aaf54 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionWorkflowImpl.java @@ -64,7 +64,7 @@ public ConnectorJobOutput run(final JobRunConfig jobRunConfig, .withFailureOrigin(connectionConfiguration.getActorType() == ActorType.SOURCE ? FailureReason.FailureOrigin.SOURCE : FailureReason.FailureOrigin.DESTINATION) .withExternalMessage("The check connection failed because of an internal error") - .withInternalMessage(e.getMessage()) + .withInternalMessage("The check connection failed because of an internal error in the scheduler used by airbyte.") .withStacktrace(e.toString())); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index 03412ca1ebb..f733e7e8564 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -115,6 +115,8 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow private static final String SYNC_TASK_QUEUE_ROUTE_RENAME_TAG = "sync_task_queue_route_rename"; private static final int GENERATE_CHECK_INPUT_CURRENT_VERSION = 1; private static final int SYNC_TASK_QUEUE_ROUTE_RENAME_CURRENT_VERSION = 1; + private static final String CHECK_WORKSPACE_TOMBSTONE_TAG = "check_workspace_tombstone"; + private static final int CHECK_WORKSPACE_TOMBSTONE_CURRENT_VERSION = 1; private final WorkflowState workflowState = new WorkflowState(UUID.randomUUID(), new NoopStateListener()); @@ -160,6 +162,11 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow @Override public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws RetryableException { try { + + if (isTombstone(connectionUpdaterInput.getConnectionId())) { + return; + } + /* * Always ensure that the connection ID is set from the input before performing any additional work. * Failure to set the connection ID before performing any work in this workflow could result in @@ -215,6 +222,16 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws Retr } } + private boolean isTombstone(UUID connectionId) { + final int checkTombstoneVersion = + Workflow.getVersion(CHECK_WORKSPACE_TOMBSTONE_TAG, Workflow.DEFAULT_VERSION, CHECK_WORKSPACE_TOMBSTONE_CURRENT_VERSION); + if (checkTombstoneVersion == Workflow.DEFAULT_VERSION || connectionId == null) { + return false; + } + + return configFetchActivity.isWorkspaceTombstone(connectionId); + } + @SuppressWarnings("PMD.UnusedLocalVariable") private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterInput connectionUpdaterInput) { return Workflow.newCancellationScope(() -> { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivity.java index e1a0da5c302..b39cc9ec23a 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivity.java @@ -79,4 +79,7 @@ class GetMaxAttemptOutput { @ActivityMethod GetMaxAttemptOutput getMaxAttempt(); + @ActivityMethod + Boolean isWorkspaceTombstone(UUID connectionId); + } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java index 5ee7d7584af..3e7ea67e593 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java @@ -24,6 +24,7 @@ import io.airbyte.api.client.model.generated.ConnectionStatus; import io.airbyte.api.client.model.generated.JobOptionalRead; import io.airbyte.api.client.model.generated.JobRead; +import io.airbyte.api.client.model.generated.WorkspaceRead; import io.airbyte.commons.temporal.exception.RetryableException; import io.airbyte.featureflag.Connection; import io.airbyte.featureflag.FeatureFlagClient; @@ -248,6 +249,17 @@ public GetMaxAttemptOutput getMaxAttempt() { return new GetMaxAttemptOutput(syncJobMaxAttempts); } + @Override + public Boolean isWorkspaceTombstone(UUID connectionId) { + try { + WorkspaceRead workspaceRead = workspaceApi.getWorkspaceByConnectionIdWithTombstone(new ConnectionIdRequestBody().connectionId(connectionId)); + return workspaceRead.getTombstone(); + } catch (ApiException e) { + log.warn("Fail to get the workspace.", e); + return false; + } + } + @Override public Optional getSourceId(final UUID connectionId) { try { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java index 02861a03e96..2999fa81cfb 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java @@ -29,14 +29,17 @@ public class GenerateInputActivityImpl implements GenerateInputActivity { private final JobsApi jobsApi; + private final PayloadChecker payloadChecker; + @SuppressWarnings("ParameterName") - public GenerateInputActivityImpl(final JobsApi jobsApi) { + public GenerateInputActivityImpl(final JobsApi jobsApi, final PayloadChecker payloadChecker) { this.jobsApi = jobsApi; + this.payloadChecker = payloadChecker; } @Override public SyncJobCheckConnectionInputs getCheckConnectionInputs(final SyncInputWithAttemptNumber input) { - return PayloadChecker.validatePayloadSize(Jsons.convertValue(AirbyteApiClient.retryWithJitter( + return payloadChecker.validatePayloadSize(Jsons.convertValue(AirbyteApiClient.retryWithJitter( () -> jobsApi.getCheckInput(new io.airbyte.api.client.model.generated.CheckInput().jobId(input.getJobId()) .attemptNumber(input.getAttemptNumber())), "Create check job input."), SyncJobCheckConnectionInputs.class)); @@ -45,7 +48,7 @@ public SyncJobCheckConnectionInputs getCheckConnectionInputs(final SyncInputWith @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) @Override public JobInput getSyncWorkflowInput(final SyncInput input) { - return PayloadChecker.validatePayloadSize(Jsons.convertValue(AirbyteApiClient.retryWithJitter( + return payloadChecker.validatePayloadSize(Jsons.convertValue(AirbyteApiClient.retryWithJitter( () -> jobsApi.getJobInput(new io.airbyte.api.client.model.generated.SyncInput().jobId(input.getJobId()) .attemptNumber(input.getAttemptId())), "Create job input."), JobInput.class)); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java index 8e8ae406bcf..72c82175152 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java @@ -79,6 +79,7 @@ public class ReplicationActivityImpl implements ReplicationActivity { private final OrchestratorHandleFactory orchestratorHandleFactory; private final MetricClient metricClient; private final FeatureFlagClient featureFlagClient; + private final PayloadChecker payloadChecker; public ReplicationActivityImpl(final SecretsRepositoryReader secretsRepositoryReader, @Named("workspaceRoot") final Path workspaceRoot, @@ -91,7 +92,8 @@ public ReplicationActivityImpl(final SecretsRepositoryReader secretsRepositoryRe final WorkloadIdGenerator workloadIdGenerator, final OrchestratorHandleFactory orchestratorHandleFactory, final MetricClient metricClient, - final FeatureFlagClient featureFlagClient) { + final FeatureFlagClient featureFlagClient, + final PayloadChecker payloadChecker) { this.replicationInputHydrator = new ReplicationInputHydrator(airbyteApiClient.getConnectionApi(), airbyteApiClient.getJobsApi(), airbyteApiClient.getStateApi(), @@ -108,6 +110,7 @@ public ReplicationActivityImpl(final SecretsRepositoryReader secretsRepositoryRe this.orchestratorHandleFactory = orchestratorHandleFactory; this.metricClient = metricClient; this.featureFlagClient = featureFlagClient; + this.payloadChecker = payloadChecker; } /** @@ -192,7 +195,7 @@ public StandardSyncOutput replicateV2(final ReplicationActivityInput replication } BackfillHelper.markBackfilledStreams(streamsToBackfill, standardSyncOutput); LOGGER.info("sync summary after backfill: {}", standardSyncOutput); - return PayloadChecker.validatePayloadSize(standardSyncOutput); + return payloadChecker.validatePayloadSize(standardSyncOutput); }, context); } diff --git a/airbyte-workload-api-server/Dockerfile b/airbyte-workload-api-server/Dockerfile index 6a11557a3a1..b305eae0188 100644 --- a/airbyte-workload-api-server/Dockerfile +++ b/airbyte-workload-api-server/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.0.1 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.1.0 FROM ${JDK_IMAGE} AS server EXPOSE 8007 5005 ENV APPLICATION airbyte-workload-api-server @@ -6,7 +6,10 @@ ENV VERSION ${VERSION} WORKDIR /app # This is automatically unzipped by Docker +USER root ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"] diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandlerImpl.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandlerImpl.kt index ae04b3307fd..d971bea4ab9 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandlerImpl.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandlerImpl.kt @@ -1,6 +1,9 @@ package io.airbyte.workload.handler import io.airbyte.config.WorkloadType +import io.airbyte.featureflag.Connection +import io.airbyte.featureflag.EnforceMutexKeyOnCreate +import io.airbyte.featureflag.FeatureFlagClient import io.airbyte.workload.api.domain.Workload import io.airbyte.workload.api.domain.WorkloadLabel import io.airbyte.workload.errors.ConflictException @@ -21,7 +24,14 @@ private val logger = KotlinLogging.logger {} @Singleton class WorkloadHandlerImpl( private val workloadRepository: WorkloadRepository, + private val featureFlagClient: FeatureFlagClient, ) : WorkloadHandler { + companion object { + val UUID_ZERO: UUID = UUID.fromString("00000000-0000-0000-0000-000000000000") + val ACTIVE_STATUSES: List = + listOf(WorkloadStatus.PENDING, WorkloadStatus.CLAIMED, WorkloadStatus.LAUNCHED, WorkloadStatus.RUNNING) + } + override fun getWorkload(workloadId: String): ApiWorkload { return getDomainWorkload(workloadId).toApi() } @@ -65,6 +75,9 @@ class WorkloadHandlerImpl( if (workloadAlreadyExists) { throw ConflictException("Workload with id: $workloadId already exists") } + + // Create the workload and then check for mutexKey uniqueness. + // This will lead to a more deterministic concurrency resolution in the event of concurrent create calls. val domainWorkload = DomainWorkload( id = workloadId, @@ -79,8 +92,27 @@ class WorkloadHandlerImpl( autoId = autoId, deadline = deadline, ) - workloadRepository.save(domainWorkload).toApi() + + // Evaluating feature flag with UUID_ZERO because the client requires a context. This feature flag is intended to be used + // as a global kill switch for validation. + if (mutexKey != null && featureFlagClient.boolVariation(EnforceMutexKeyOnCreate, Connection(UUID_ZERO))) { + // Keep the most recent workload by creation date with mutexKey, fail the others. + workloadRepository + .searchByMutexKeyAndStatusInList(mutexKey, statuses = ACTIVE_STATUSES) + .sortedByDescending { it.createdAt } + .drop(1) + .forEach { + try { + logger.info { "${it.id} violates the $mutexKey uniqueness constraint, failing in favor of $workloadId before continuing." } + failWorkload(it.id, source = "workload-api", reason = "Superseded by $workloadId") + } catch (_: InvalidStatusTransitionException) { + // This edge case happens if the workload reached a terminal state through another path. + // This would be unusual but not actionable so we're logging a message rather than failing the call. + logger.info { "${it.id} was completed before being superseded by $workloadId" } + } + } + } } override fun claimWorkload( diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/WorkloadRepository.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/WorkloadRepository.kt index ea28f7f55a2..a553a3e2fdc 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/WorkloadRepository.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/WorkloadRepository.kt @@ -48,6 +48,11 @@ interface WorkloadRepository : PageableRepository { deadline: OffsetDateTime, ): List + fun searchByMutexKeyAndStatusInList( + mutexKey: String, + statuses: List, + ): List + @Query( """ SELECT * FROM workload diff --git a/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/handler/WorkloadHandlerImplTest.kt b/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/handler/WorkloadHandlerImplTest.kt index bc8947ed247..2f02231d8e6 100644 --- a/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/handler/WorkloadHandlerImplTest.kt +++ b/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/handler/WorkloadHandlerImplTest.kt @@ -1,5 +1,6 @@ package io.airbyte.workload.handler +import io.airbyte.featureflag.TestClient import io.airbyte.workload.api.domain.WorkloadLabel import io.airbyte.workload.errors.ConflictException import io.airbyte.workload.errors.InvalidStatusTransitionException @@ -40,6 +41,17 @@ class WorkloadHandlerImplTest { every { workloadHandler.offsetDateTime() }.returns(now) } + @Test + fun `test active statuses are complete`() { + assertEquals( + setOf(WorkloadStatus.PENDING, WorkloadStatus.CLAIMED, WorkloadStatus.LAUNCHED, WorkloadStatus.RUNNING), + WorkloadHandlerImpl.ACTIVE_STATUSES.toSet(), + ) + assertFalse(WorkloadHandlerImpl.ACTIVE_STATUSES.contains(WorkloadStatus.CANCELLED)) + assertFalse(WorkloadHandlerImpl.ACTIVE_STATUSES.contains(WorkloadStatus.FAILURE)) + assertFalse(WorkloadHandlerImpl.ACTIVE_STATUSES.contains(WorkloadStatus.SUCCESS)) + } + @Test fun `test get workload`() { val domainWorkload = @@ -74,6 +86,7 @@ class WorkloadHandlerImplTest { val workloadLabels = mutableListOf(workloadLabel1, workloadLabel2) every { workloadRepository.existsById(WORKLOAD_ID) }.returns(false) + every { workloadRepository.searchByMutexKeyAndStatusInList("mutex-this", WorkloadHandlerImpl.ACTIVE_STATUSES) }.returns(listOf()) every { workloadRepository.save(any()) }.returns( Fixtures.workload(), ) @@ -118,6 +131,43 @@ class WorkloadHandlerImplTest { } } + @Test + fun `test create workload mutex conflict`() { + val workloadIdWithSuccessfulFail = "workload-id-with-successful-fail" + val workloadIdWithFailedFail = "workload-id-with-failed-fail" + val duplWorkloads = + listOf( + Fixtures.workload(workloadIdWithSuccessfulFail, createdAt = OffsetDateTime.now().minusSeconds(5)), + Fixtures.workload(workloadIdWithFailedFail, createdAt = OffsetDateTime.now().minusSeconds(10)), + ) + val newWorkload = Fixtures.workload(WORKLOAD_ID) + every { workloadRepository.existsById(WORKLOAD_ID) }.returns(false) + every { + workloadHandler.failWorkload(workloadIdWithSuccessfulFail, any(), any()) + }.answers {} + every { + workloadHandler.failWorkload(workloadIdWithFailedFail, any(), any()) + }.throws(InvalidStatusTransitionException("$workloadIdWithFailedFail")) + every { workloadRepository.save(any()) }.returns(newWorkload) + every { + workloadRepository.searchByMutexKeyAndStatusInList( + "mutex-this", + WorkloadHandlerImpl.ACTIVE_STATUSES, + ) + }.returns(duplWorkloads + listOf(newWorkload)) + + workloadHandler.createWorkload(WORKLOAD_ID, null, "", "", "US", "mutex-this", io.airbyte.config.WorkloadType.SYNC, UUID.randomUUID(), now) + verify { + workloadHandler.failWorkload(workloadIdWithFailedFail, any(), any()) + workloadHandler.failWorkload(workloadIdWithSuccessfulFail, any(), any()) + workloadRepository.save( + match { + it.id == WORKLOAD_ID && it.mutexKey == "mutex-this" + }, + ) + } + } + @Test fun `test get workloads`() { val domainWorkload = @@ -592,7 +642,7 @@ class WorkloadHandlerImplTest { @Test fun `offsetDateTime method should always return current time`() { - val workloadHandlerImpl = WorkloadHandlerImpl(mockk()) + val workloadHandlerImpl = WorkloadHandlerImpl(mockk(), TestClient()) val offsetDateTime = workloadHandlerImpl.offsetDateTime() Thread.sleep(10) val offsetDateTimeAfter10Ms = workloadHandlerImpl.offsetDateTime() @@ -603,7 +653,7 @@ class WorkloadHandlerImplTest { val workloadRepository = mockk() const val WORKLOAD_ID = "test" const val DATAPLANE_ID = "dataplaneId" - val workloadHandler = spyk(WorkloadHandlerImpl(workloadRepository)) + val workloadHandler = spyk(WorkloadHandlerImpl(workloadRepository, TestClient(mapOf("platform.enforce-mutex-key-on-create" to true)))) fun workload( id: String = WORKLOAD_ID, @@ -615,6 +665,7 @@ class WorkloadHandlerImplTest { geography: String = "US", mutexKey: String = "", type: WorkloadType = WorkloadType.SYNC, + createdAt: OffsetDateTime = OffsetDateTime.now(), ): Workload = Workload( id = id, @@ -626,6 +677,7 @@ class WorkloadHandlerImplTest { geography = geography, mutexKey = mutexKey, type = type, + createdAt = createdAt, ) } } diff --git a/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/repository/WorkloadRepositoryTest.kt b/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/repository/WorkloadRepositoryTest.kt index aa785e9aff7..564440b8fbe 100644 --- a/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/repository/WorkloadRepositoryTest.kt +++ b/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/repository/WorkloadRepositoryTest.kt @@ -241,6 +241,28 @@ internal class WorkloadRepositoryTest { assertEquals("dataplaneId2", persistedWorkload.get().dataplaneId) } + @Test + fun `test mutex search`() { + val mutexKey = "mutex-search-test" + val workload1 = + Fixtures.workload( + id = "workload-mutex-search-1", + status = WorkloadStatus.PENDING, + mutexKey = mutexKey, + ) + workloadRepo.save(workload1) + + val match = workloadRepo.searchByMutexKeyAndStatusInList(mutexKey, listOf(WorkloadStatus.PENDING, WorkloadStatus.RUNNING)) + assertEquals(1, match.size) + assertEquals(workload1.id, match[0].id) + + val emptyResult = workloadRepo.searchByMutexKeyAndStatusInList(mutexKey, listOf(WorkloadStatus.CLAIMED, WorkloadStatus.RUNNING)) + assertEquals(0, emptyResult.size) + + val mutexMismatch = workloadRepo.searchByMutexKeyAndStatusInList("mismatch", listOf(WorkloadStatus.PENDING, WorkloadStatus.RUNNING)) + assertEquals(0, mutexMismatch.size) + } + @Test fun `test search`() { val workload1 = diff --git a/airbyte-workload-launcher/Dockerfile b/airbyte-workload-launcher/Dockerfile index 74c296b13d0..b03fcc59680 100644 --- a/airbyte-workload-launcher/Dockerfile +++ b/airbyte-workload-launcher/Dockerfile @@ -1,4 +1,5 @@ -FROM airbyte/airbyte-base-java-worker-image:2.0.1 +ARG JAVA_WORKER_BASE_IMAGE_VERSION=2.1.0 +FROM airbyte/airbyte-base-java-worker-image:${JAVA_WORKER_BASE_IMAGE_VERSION} ENV APPLICATION airbyte-workload-launcher ENV VERSION ${VERSION} @@ -6,7 +7,10 @@ ENV VERSION ${VERSION} WORKDIR /app # This is automatically unzipped by Docker +USER root ADD airbyte-app.tar /app +RUN chown -R airbyte:airbyte /app +USER airbyte:airbyte # 8016 is the port micronaut listens on # 5005 is the remote debug port diff --git a/airbyte-workload-launcher/src/main/kotlin/config/EnvVarConfigBeanFactory.kt b/airbyte-workload-launcher/src/main/kotlin/config/EnvVarConfigBeanFactory.kt index 9ac8517862f..65bf4e2b5a2 100644 --- a/airbyte-workload-launcher/src/main/kotlin/config/EnvVarConfigBeanFactory.kt +++ b/airbyte-workload-launcher/src/main/kotlin/config/EnvVarConfigBeanFactory.kt @@ -4,20 +4,13 @@ package io.airbyte.workload.launcher.config -import io.airbyte.commons.constants.WorkerConstants -import io.airbyte.commons.features.EnvVariableFeatureFlags -import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.workers.config.WorkerConfigs -import io.airbyte.config.Configs -import io.airbyte.config.EnvConfigs import io.airbyte.config.storage.StorageConfig import io.airbyte.workers.process.Metadata.AWS_ACCESS_KEY_ID import io.airbyte.workers.process.Metadata.AWS_SECRET_ACCESS_KEY -import io.airbyte.workers.sync.OrchestratorConstants import io.fabric8.kubernetes.api.model.EnvVar import io.fabric8.kubernetes.api.model.EnvVarSource import io.fabric8.kubernetes.api.model.SecretKeySelector -import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Value import io.micronaut.context.env.Environment @@ -25,109 +18,12 @@ import io.micronaut.core.util.StringUtils import jakarta.inject.Named import jakarta.inject.Singleton import java.util.function.Consumer -import io.airbyte.commons.envvar.EnvVar as AbEnvVar - -private val logger = KotlinLogging.logger {} /** * Provides and configures the environment variables for the containers we launch. */ @Factory class EnvVarConfigBeanFactory { - /** - * Map of env vars to be passed to the orchestrator container. - */ - @Singleton - @Named("orchestratorEnvMap") - fun orchestratorEnvMap( - featureFlags: FeatureFlags, - workerEnv: Configs.WorkerEnvironment, - storageConfig: StorageConfig, - @Named("workloadApiEnvMap") workloadApiEnvMap: Map, - @Named("metricsEnvMap") metricsEnvMap: Map, - @Named("micronautEnvMap") micronautEnvMap: Map, - @Named("apiClientEnvMap") apiClientEnvMap: Map, - @Value("\${airbyte.container.orchestrator.java-opts}") containerOrchestratorJavaOpts: String, - @Value("\${airbyte.connector.source.credentials.aws.assumed-role.secret-name}") awsAssumedRoleSecretName: String, - ): Map { - // Build the map of additional environment variables to be passed to the container orchestrator - val envMap: MutableMap = HashMap() - envMap[EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA] = java.lang.Boolean.toString(featureFlags.autoDetectSchema()) - envMap[EnvVariableFeatureFlags.APPLY_FIELD_SELECTION] = java.lang.Boolean.toString(featureFlags.applyFieldSelection()) - envMap[EnvVariableFeatureFlags.FIELD_SELECTION_WORKSPACES] = featureFlags.fieldSelectionWorkspaces() - envMap[JAVA_OPTS_ENV_VAR] = containerOrchestratorJavaOpts - - val configs: Configs = EnvConfigs() - envMap[AbEnvVar.FEATURE_FLAG_CLIENT.name] = AbEnvVar.FEATURE_FLAG_CLIENT.fetch() ?: "" - envMap[AbEnvVar.LAUNCHDARKLY_KEY.name] = AbEnvVar.LAUNCHDARKLY_KEY.fetch() ?: "" - envMap[AbEnvVar.OTEL_COLLECTOR_ENDPOINT.name] = AbEnvVar.OTEL_COLLECTOR_ENDPOINT.fetch() ?: "" - envMap[AbEnvVar.SOCAT_KUBE_CPU_LIMIT.name] = configs.socatSidecarKubeCpuLimit - envMap[AbEnvVar.SOCAT_KUBE_CPU_REQUEST.name] = configs.socatSidecarKubeCpuRequest - - // secret name used by orchestrator for assumed role look-ups - envMap[AbEnvVar.AWS_ASSUME_ROLE_SECRET_NAME.name] = awsAssumedRoleSecretName - - // Manually add the worker environment - envMap[WorkerConstants.WORKER_ENVIRONMENT] = workerEnv.name - - // Cloud storage config - envMap.putAll(storageConfig.toEnvVarMap()) - - // Workload Api configuration - envMap.putAll(workloadApiEnvMap) - - // Api client configuration - envMap.putAll(apiClientEnvMap) - - // Metrics configuration - envMap.putAll(metricsEnvMap) - - // Micronaut environment - envMap.putAll(micronautEnvMap) - - // TODO: Don't do this. Be explicit about what env vars we pass. - // Copy over all local values - val localEnvMap = - System.getenv() - .filter { OrchestratorConstants.ENV_VARS_TO_TRANSFER.contains(it.key) } - - envMap.putAll(localEnvMap) - - return envMap - } - - /** - * The list of environment variables to be passed to the orchestrator. - * The created list contains both regular environment variables and environment variables that - * are sourced from Kubernetes secrets. - */ - @Singleton - @Named("orchestratorEnvVars") - fun orchestratorEnvVars( - @Named("orchestratorEnvMap") envMap: Map, - @Named("orchestratorSecretsEnvMap") secretsEnvMap: Map, - ): List { - val secretEnvVars = - secretsEnvMap - .map { EnvVar(it.key, null, it.value) } - .toList() - - val envVars = - envMap - .filterNot { env -> - secretsEnvMap.containsKey(env.key) - .also { - if (it) { - logger.info { "Skipping env-var ${env.key} as it was already defined as a secret. " } - } - } - } - .map { EnvVar(it.key, it.value, null) } - .toList() - - return envVars + secretEnvVars - } - /** * The list of env vars to be passed to the check sidecar container. */ @@ -397,7 +293,7 @@ class EnvVarConfigBeanFactory { private const val DD_ENV_ENV_VAR = "DD_ENV" private const val DD_SERVICE_ENV_VAR = "DD_SERVICE" private const val DD_VERSION_ENV_VAR = "DD_VERSION" - private const val JAVA_OPTS_ENV_VAR = "JAVA_OPTS" + const val JAVA_OPTS_ENV_VAR = "JAVA_OPTS" private const val PUBLISH_METRICS_ENV_VAR = "PUBLISH_METRICS" private const val CONTROL_PLANE_AUTH_ENDPOINT_ENV_VAR = "CONTROL_PLANE_AUTH_ENDPOINT" private const val DATA_PLANE_SERVICE_ACCOUNT_CREDENTIALS_PATH_ENV_VAR = "DATA_PLANE_SERVICE_ACCOUNT_CREDENTIALS_PATH" diff --git a/airbyte-workload-launcher/src/main/kotlin/config/OrchestratorEnvSingleton.kt b/airbyte-workload-launcher/src/main/kotlin/config/OrchestratorEnvSingleton.kt new file mode 100644 index 00000000000..30d7cd2a121 --- /dev/null +++ b/airbyte-workload-launcher/src/main/kotlin/config/OrchestratorEnvSingleton.kt @@ -0,0 +1,129 @@ +package io.airbyte.workload.launcher.config + +import io.airbyte.commons.constants.WorkerConstants +import io.airbyte.commons.features.EnvVariableFeatureFlags +import io.airbyte.commons.features.FeatureFlags +import io.airbyte.config.Configs +import io.airbyte.config.EnvConfigs +import io.airbyte.config.storage.StorageConfig +import io.airbyte.featureflag.Connection +import io.airbyte.featureflag.ContainerOrchestratorJavaOpts +import io.airbyte.featureflag.FeatureFlagClient +import io.airbyte.workers.sync.OrchestratorConstants +import io.fabric8.kubernetes.api.model.EnvVar +import io.fabric8.kubernetes.api.model.EnvVarSource +import io.github.oshai.kotlinlogging.KotlinLogging +import io.micronaut.context.annotation.Value +import jakarta.inject.Named +import jakarta.inject.Singleton +import java.util.UUID +import io.airbyte.commons.envvar.EnvVar as AbEnvVar + +private val logger = KotlinLogging.logger {} + +@Singleton +class OrchestratorEnvSingleton( + private val featureFlagClient: FeatureFlagClient, + private val featureFlags: FeatureFlags, + private val workerEnv: Configs.WorkerEnvironment, + private val storageConfig: StorageConfig, + @Named("workloadApiEnvMap") private val workloadApiEnvMap: Map, + @Named("metricsEnvMap") private val metricsEnvMap: Map, + @Named("micronautEnvMap") private val micronautEnvMap: Map, + @Named("apiClientEnvMap") private val apiClientEnvMap: Map, + @Value("\${airbyte.container.orchestrator.java-opts}") private val containerOrchestratorJavaOpts: String, + @Value("\${airbyte.connector.source.credentials.aws.assumed-role.secret-name}") private val awsAssumedRoleSecretName: String, + @Named("orchestratorSecretsEnvMap") private val secretsEnvMap: Map, +) { + /** + * Map of env vars to be passed to the orchestrator container. + */ + fun orchestratorEnvMap(connectionId: UUID): Map { + // Build the map of additional environment variables to be passed to the container orchestrator + val envMap: MutableMap = HashMap() + envMap[EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA] = java.lang.Boolean.toString(featureFlags.autoDetectSchema()) + envMap[EnvVariableFeatureFlags.APPLY_FIELD_SELECTION] = java.lang.Boolean.toString(featureFlags.applyFieldSelection()) + envMap[EnvVariableFeatureFlags.FIELD_SELECTION_WORKSPACES] = featureFlags.fieldSelectionWorkspaces() + overrideOrchestratorJavaOpts(envMap, connectionId) + val configs: Configs = EnvConfigs() + envMap[AbEnvVar.FEATURE_FLAG_CLIENT.name] = AbEnvVar.FEATURE_FLAG_CLIENT.fetch() ?: "" + envMap[AbEnvVar.LAUNCHDARKLY_KEY.name] = AbEnvVar.LAUNCHDARKLY_KEY.fetch() ?: "" + envMap[AbEnvVar.OTEL_COLLECTOR_ENDPOINT.name] = AbEnvVar.OTEL_COLLECTOR_ENDPOINT.fetch() ?: "" + envMap[AbEnvVar.SOCAT_KUBE_CPU_LIMIT.name] = configs.socatSidecarKubeCpuLimit + envMap[AbEnvVar.SOCAT_KUBE_CPU_REQUEST.name] = configs.socatSidecarKubeCpuRequest + + // secret name used by orchestrator for assumed role look-ups + envMap[AbEnvVar.AWS_ASSUME_ROLE_SECRET_NAME.name] = awsAssumedRoleSecretName + + // Manually add the worker environment + envMap[WorkerConstants.WORKER_ENVIRONMENT] = workerEnv.name + + // Cloud storage config + envMap.putAll(storageConfig.toEnvVarMap()) + + // Workload Api configuration + envMap.putAll(workloadApiEnvMap) + + // Api client configuration + envMap.putAll(apiClientEnvMap) + + // Metrics configuration + envMap.putAll(metricsEnvMap) + + // Micronaut environment + envMap.putAll(micronautEnvMap) + + // TODO: Don't do this. Be explicit about what env vars we pass. + // Copy over all local values + val localEnvMap = + System.getenv() + .filter { OrchestratorConstants.ENV_VARS_TO_TRANSFER.contains(it.key) } + + envMap.putAll(localEnvMap) + + return envMap + } + + private fun overrideOrchestratorJavaOpts( + envMap: MutableMap, + connectionId: UUID, + ) { + val injectedJavaOpts: String = featureFlagClient.stringVariation(ContainerOrchestratorJavaOpts, Connection(connectionId)) + if (injectedJavaOpts.isNotEmpty()) { + envMap[EnvVarConfigBeanFactory.JAVA_OPTS_ENV_VAR] = injectedJavaOpts.trim() + } else { + envMap[EnvVarConfigBeanFactory.JAVA_OPTS_ENV_VAR] = containerOrchestratorJavaOpts + } + } + + /** + * The list of environment variables to be passed to the orchestrator. + * The created list contains both regular environment variables and environment variables that + * are sourced from Kubernetes secrets. + */ + fun orchestratorEnvVars(connectionId: UUID): List { + val secretEnvVars = + secretEnvMap() + .map { EnvVar(it.key, null, it.value) } + .toList() + + val envVars = + orchestratorEnvMap(connectionId) + .filterNot { env -> + secretEnvMap().containsKey(env.key) + .also { + if (it) { + logger.info { "Skipping env-var ${env.key} as it was already defined as a secret. " } + } + } + } + .map { EnvVar(it.key, it.value, null) } + .toList() + + return envVars + secretEnvVars + } + + fun secretEnvMap(): Map { + return secretsEnvMap + } +} diff --git a/airbyte-workload-launcher/src/main/kotlin/pods/DockerPodClient.kt b/airbyte-workload-launcher/src/main/kotlin/pods/DockerPodClient.kt index 3d717fe684d..42d22e9e939 100644 --- a/airbyte-workload-launcher/src/main/kotlin/pods/DockerPodClient.kt +++ b/airbyte-workload-launcher/src/main/kotlin/pods/DockerPodClient.kt @@ -11,6 +11,7 @@ import io.airbyte.workers.process.KubeContainerInfo import io.airbyte.workers.process.KubePodInfo import io.airbyte.workers.sync.OrchestratorConstants import io.airbyte.workers.sync.ReplicationLauncherWorker +import io.airbyte.workload.launcher.config.OrchestratorEnvSingleton import io.airbyte.workload.launcher.model.getAttemptId import io.airbyte.workload.launcher.model.getJobId import io.airbyte.workload.launcher.model.getOrchestratorResourceReqs @@ -30,7 +31,7 @@ import java.util.UUID @Requires(notEnv = [Environment.KUBERNETES]) class DockerPodClient( private val serializer: ObjectSerializer, - @Named("orchestratorEnvMap") private val orchestratorEnvMap: Map, + private val orchestratorEnvSingleton: OrchestratorEnvSingleton, private val podLauncher: DockerPodLauncher, @Named("orchestratorKubeContainerInfo") private val orchestratorInfo: KubeContainerInfo, private val podNameGenerator: PodNameGenerator, @@ -47,7 +48,7 @@ class DockerPodClient( name = launcherInput.autoId.toString(), imageName = orchestratorInfo.image, mutex = launcherInput.mutexKey, - envMap = orchestratorEnvMap, + envMap = orchestratorEnvSingleton.orchestratorEnvMap(replicationInput.connectionId), fileMap = buildFileMap(launcherInput.workloadId, replicationInput, replicationInput.jobRunConfig), orchestratorReqs = replicationInput.getOrchestratorResourceReqs(), ) @@ -88,7 +89,7 @@ class DockerPodClient( jobRunConfig: JobRunConfig, ): Map { return mapOf( - OrchestratorConstants.INIT_FILE_ENV_MAP to serializer.serialize(orchestratorEnvMap), + OrchestratorConstants.INIT_FILE_ENV_MAP to serializer.serialize(orchestratorEnvSingleton.orchestratorEnvMap(input.connectionId)), OrchestratorConstants.INIT_FILE_JOB_RUN_CONFIG to serializer.serialize(jobRunConfig), AsyncOrchestratorPodProcess.KUBE_POD_INFO to serializer.serialize( diff --git a/airbyte-workload-launcher/src/main/kotlin/pods/DockerPodLauncher.kt b/airbyte-workload-launcher/src/main/kotlin/pods/DockerPodLauncher.kt index 8fd0926a39c..bfce72d325a 100644 --- a/airbyte-workload-launcher/src/main/kotlin/pods/DockerPodLauncher.kt +++ b/airbyte-workload-launcher/src/main/kotlin/pods/DockerPodLauncher.kt @@ -25,7 +25,7 @@ data class DockerConfig( @Value("\${airbyte.docker.network}") val dockerNetwork: String, @Value("\${airbyte.docker.workspace-mount-name}") val workspaceMountName: String, @Value("\${airbyte.docker.workspace-mount-path}") val workspaceMountPath: Path, - @Value("\${airbyte.docker.docker-socket}") val dockerSocket: Path, + @Value("\${airbyte.docker.docker-socket}") val dockerSocket: Path?, @Value("\${airbyte.docker.local-mount-name}") val localMountName: String, @Value("\${airbyte.docker.local-mount-path}") val localMountPath: Path, ) @@ -91,7 +91,9 @@ class DockerPodLauncher(private val dockerConfig: DockerConfig) { // the ':' syntax specifies the volume on the local instance to mount to the container // e.g. :. Not be confused with Micronaut's // default value syntax. - cmd.addOption("-v", "${dockerConfig.dockerSocket}:/var/run/docker.sock") + dockerConfig.dockerSocket?.let { + cmd.addOption("-v", "$it:/var/run/docker.sock") + } podConfig.mutex?.let { cmd.addOption("--label", "mutex=$it") diff --git a/airbyte-workload-launcher/src/main/kotlin/pods/KubePodClient.kt b/airbyte-workload-launcher/src/main/kotlin/pods/KubePodClient.kt index cfcf94b0d76..8ecc2229423 100644 --- a/airbyte-workload-launcher/src/main/kotlin/pods/KubePodClient.kt +++ b/airbyte-workload-launcher/src/main/kotlin/pods/KubePodClient.kt @@ -3,8 +3,6 @@ package io.airbyte.workload.launcher.pods import com.google.common.annotations.VisibleForTesting import datadog.trace.api.Trace import io.airbyte.commons.constants.WorkerConstants.KubeConstants.FULL_POD_TIMEOUT -import io.airbyte.featureflag.Connection -import io.airbyte.featureflag.ContainerOrchestratorJavaOpts import io.airbyte.featureflag.FeatureFlagClient import io.airbyte.metrics.lib.ApmTraceUtils import io.airbyte.persistence.job.models.ReplicationInput @@ -63,16 +61,15 @@ class KubePodClient( val kubeInput = mapper.toKubeInput(launcherInput.workloadId, inputWithLabels, sharedLabels) - val injectedJavaOpts: String = featureFlagClient.stringVariation(ContainerOrchestratorJavaOpts, Connection(replicationInput.connectionId)) - val additionalEnvVars = if (injectedJavaOpts.isNotEmpty()) mapOf("JAVA_OPTS" to injectedJavaOpts) else mapOf() var pod = orchestratorPodFactory.create( + replicationInput.connectionId, kubeInput.orchestratorLabels, kubeInput.resourceReqs, kubeInput.nodeSelectors, kubeInput.kubePodInfo, kubeInput.annotations, - additionalEnvVars, + mapOf(), ) try { pod = diff --git a/airbyte-workload-launcher/src/main/kotlin/pods/PayloadKubeInputMapper.kt b/airbyte-workload-launcher/src/main/kotlin/pods/PayloadKubeInputMapper.kt index f5af951e69a..8aafd1f76cc 100644 --- a/airbyte-workload-launcher/src/main/kotlin/pods/PayloadKubeInputMapper.kt +++ b/airbyte-workload-launcher/src/main/kotlin/pods/PayloadKubeInputMapper.kt @@ -25,6 +25,7 @@ import io.airbyte.workers.sync.OrchestratorConstants import io.airbyte.workers.sync.ReplicationLauncherWorker.INIT_FILE_DESTINATION_LAUNCHER_CONFIG import io.airbyte.workers.sync.ReplicationLauncherWorker.INIT_FILE_SOURCE_LAUNCHER_CONFIG import io.airbyte.workers.sync.ReplicationLauncherWorker.REPLICATION +import io.airbyte.workload.launcher.config.OrchestratorEnvSingleton import io.airbyte.workload.launcher.model.getAttemptId import io.airbyte.workload.launcher.model.getJobId import io.airbyte.workload.launcher.model.getOrchestratorResourceReqs @@ -44,9 +45,9 @@ class PayloadKubeInputMapper( private val serializer: ObjectSerializer, private val labeler: PodLabeler, private val podNameGenerator: PodNameGenerator, + private val orchestratorEnvSingleton: OrchestratorEnvSingleton, @Value("\${airbyte.worker.job.kube.namespace}") private val namespace: String?, @Named("orchestratorKubeContainerInfo") private val orchestratorKubeContainerInfo: KubeContainerInfo, - @Named("orchestratorEnvMap") private val envMap: Map, @Named("connectorAwsAssumedRoleSecretEnv") private val connectorAwsAssumedRoleSecretEnvList: List, @Named("replicationWorkerConfigs") private val replicationWorkerConfigs: WorkerConfigs, @Named("checkWorkerConfigs") private val checkWorkerConfigs: WorkerConfigs, @@ -255,7 +256,7 @@ class PayloadKubeInputMapper( mapOf( OrchestratorConstants.INIT_FILE_INPUT to serializer.serialize(input), OrchestratorConstants.INIT_FILE_APPLICATION to REPLICATION, - OrchestratorConstants.INIT_FILE_ENV_MAP to serializer.serialize(envMap), + OrchestratorConstants.INIT_FILE_ENV_MAP to serializer.serialize(orchestratorEnvSingleton.orchestratorEnvMap(input.connectionId)), OrchestratorConstants.WORKLOAD_ID_FILE to workloadId, INIT_FILE_SOURCE_LAUNCHER_CONFIG to serializer.serialize(input.sourceLauncherConfig), INIT_FILE_DESTINATION_LAUNCHER_CONFIG to serializer.serialize(input.destinationLauncherConfig), diff --git a/airbyte-workload-launcher/src/main/kotlin/pods/factories/OrchestratorPodFactory.kt b/airbyte-workload-launcher/src/main/kotlin/pods/factories/OrchestratorPodFactory.kt index 3d9acf200ef..5530540a6b9 100644 --- a/airbyte-workload-launcher/src/main/kotlin/pods/factories/OrchestratorPodFactory.kt +++ b/airbyte-workload-launcher/src/main/kotlin/pods/factories/OrchestratorPodFactory.kt @@ -7,6 +7,7 @@ import io.airbyte.featureflag.FeatureFlagClient import io.airbyte.featureflag.UseCustomK8sScheduler import io.airbyte.workers.process.KubePodInfo import io.airbyte.workers.process.KubePodProcess +import io.airbyte.workload.launcher.config.OrchestratorEnvSingleton import io.fabric8.kubernetes.api.model.ContainerBuilder import io.fabric8.kubernetes.api.model.ContainerPort import io.fabric8.kubernetes.api.model.EnvVar @@ -17,17 +18,19 @@ import io.fabric8.kubernetes.api.model.VolumeMount import io.micronaut.context.annotation.Value import jakarta.inject.Named import jakarta.inject.Singleton +import java.util.UUID @Singleton class OrchestratorPodFactory( private val featureFlagClient: FeatureFlagClient, + private val orchestratorEnvSingleton: OrchestratorEnvSingleton, @Value("\${airbyte.worker.job.kube.serviceAccount}") private val serviceAccount: String?, - @Named("orchestratorEnvVars") private val sharedEnvVars: List, @Named("orchestratorContainerPorts") private val containerPorts: List, private val volumeFactory: VolumeFactory, private val initContainerFactory: InitContainerFactory, ) { fun create( + connectionId: UUID, allLabels: Map, resourceRequirements: ResourceRequirements?, nodeSelectors: Map, @@ -66,7 +69,7 @@ class OrchestratorPodFactory( .withImage(kubePodInfo.mainContainerInfo.image) .withImagePullPolicy(kubePodInfo.mainContainerInfo.pullPolicy) .withResources(containerResources) - .withEnv(sharedEnvVars + extraKubeEnv) + .withEnv(orchestratorEnvSingleton.orchestratorEnvVars(connectionId) + extraKubeEnv) .withPorts(containerPorts) .withVolumeMounts(volumeMounts) .build() diff --git a/airbyte-workload-launcher/src/main/resources/application.yml b/airbyte-workload-launcher/src/main/resources/application.yml index 9c4d151b707..94e1ff302b2 100644 --- a/airbyte-workload-launcher/src/main/resources/application.yml +++ b/airbyte-workload-launcher/src/main/resources/application.yml @@ -51,7 +51,7 @@ airbyte: max: ${KUBERNETES_CLIENT_MAX_RETRIES:5} resource-check-rate: ${WORKLOAD_LAUNCHER_POD_PENDING_RESOURCE_CHECK_RATE:PT30S} docker: - docker-socket: ${DOCKER_SOCKET:/var/run/docker.sock} + docker-socket: ${DOCKER_SOCKET:} network: ${DOCKER_NETWORK:host} workspace-mount-name: ${WORKSPACE_DOCKER_MOUNT:} workspace-mount-path: ${WORKSPACE_ROOT} diff --git a/airbyte-workload-launcher/src/test/kotlin/config/EnvVarConfigBeanFactoryTest.kt b/airbyte-workload-launcher/src/test/kotlin/config/EnvVarConfigBeanFactoryTest.kt index 5418374146c..9f1736cdb27 100644 --- a/airbyte-workload-launcher/src/test/kotlin/config/EnvVarConfigBeanFactoryTest.kt +++ b/airbyte-workload-launcher/src/test/kotlin/config/EnvVarConfigBeanFactoryTest.kt @@ -10,11 +10,15 @@ import io.airbyte.workload.launcher.config.EnvVarConfigBeanFactory import io.airbyte.workload.launcher.config.EnvVarConfigBeanFactory.Companion.AWS_ASSUME_ROLE_ACCESS_KEY_ID_ENV_VAR import io.airbyte.workload.launcher.config.EnvVarConfigBeanFactory.Companion.AWS_ASSUME_ROLE_SECRET_ACCESS_KEY_ENV_VAR import io.airbyte.workload.launcher.config.EnvVarConfigBeanFactory.Companion.WORKLOAD_API_BEARER_TOKEN_ENV_VAR +import io.airbyte.workload.launcher.config.OrchestratorEnvSingleton import io.fabric8.kubernetes.api.model.EnvVarSource import io.fabric8.kubernetes.api.model.SecretKeySelector +import io.mockk.every +import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test +import java.util.UUID class EnvVarConfigBeanFactoryTest { companion object { @@ -114,11 +118,19 @@ class EnvVarConfigBeanFactoryTest { @Test fun `test final env vars contain secret env vars an non-secret env vars`() { - val factory = EnvVarConfigBeanFactory() + val factory: OrchestratorEnvSingleton = mockk() + every { factory.orchestratorEnvMap(any()) } returns (mapOf(Pair(ENV_VAR_NAME1, ENV_VAR_VALUE1), Pair(ENV_VAR_NAME2, ENV_VAR_VALUE2))) + every { + factory.secretEnvMap() + } returns ( + mapOf( + Pair(ENV_VAR_NAME3, EnvVarSource(null, null, null, SecretKeySelector(BEARER_TOKEN_SECRET_KEY, BEARER_TOKEN_SECRET_NAME, false))), + ) + ) + every { factory.orchestratorEnvVars(any()) } answers { callOriginal() } val orchestratorEnvVars = factory.orchestratorEnvVars( - mapOf(Pair(ENV_VAR_NAME1, ENV_VAR_VALUE1), Pair(ENV_VAR_NAME2, ENV_VAR_VALUE2)), - mapOf(Pair(ENV_VAR_NAME3, EnvVarSource(null, null, null, SecretKeySelector(BEARER_TOKEN_SECRET_KEY, BEARER_TOKEN_SECRET_NAME, false)))), + UUID.randomUUID(), ).sortedBy { it.name } assertEquals(ENV_VAR_NAME1, orchestratorEnvVars[0].name) diff --git a/airbyte-workload-launcher/src/test/kotlin/pods/KubePodClientTest.kt b/airbyte-workload-launcher/src/test/kotlin/pods/KubePodClientTest.kt index 6b7a82d0396..ce6d124a5a5 100644 --- a/airbyte-workload-launcher/src/test/kotlin/pods/KubePodClientTest.kt +++ b/airbyte-workload-launcher/src/test/kotlin/pods/KubePodClientTest.kt @@ -137,6 +137,7 @@ class KubePodClientTest { every { orchestratorPodFactory.create( + any(), replKubeInput.orchestratorLabels, replKubeInput.resourceReqs, replKubeInput.nodeSelectors, @@ -175,6 +176,7 @@ class KubePodClientTest { every { orchestratorPodFactory.create( + any(), replKubeInput.orchestratorLabels, replKubeInput.resourceReqs, replKubeInput.nodeSelectors, @@ -210,6 +212,7 @@ class KubePodClientTest { every { orchestratorPodFactory.create( + any(), replKubeInput.orchestratorLabels, replKubeInput.resourceReqs, replKubeInput.nodeSelectors, diff --git a/airbyte-workload-launcher/src/test/kotlin/pods/PayloadKubeInputMapperTest.kt b/airbyte-workload-launcher/src/test/kotlin/pods/PayloadKubeInputMapperTest.kt index 68c5be29806..aaf954b44dd 100644 --- a/airbyte-workload-launcher/src/test/kotlin/pods/PayloadKubeInputMapperTest.kt +++ b/airbyte-workload-launcher/src/test/kotlin/pods/PayloadKubeInputMapperTest.kt @@ -24,6 +24,7 @@ import io.airbyte.workers.sync.OrchestratorConstants import io.airbyte.workers.sync.ReplicationLauncherWorker import io.airbyte.workers.sync.ReplicationLauncherWorker.INIT_FILE_DESTINATION_LAUNCHER_CONFIG import io.airbyte.workers.sync.ReplicationLauncherWorker.INIT_FILE_SOURCE_LAUNCHER_CONFIG +import io.airbyte.workload.launcher.config.OrchestratorEnvSingleton import io.airbyte.workload.launcher.model.getActorType import io.airbyte.workload.launcher.model.getAttemptId import io.airbyte.workload.launcher.model.getJobId @@ -52,7 +53,9 @@ class PayloadKubeInputMapperTest { val namespace = "test-namespace" val podNameGenerator = PodNameGenerator(namespace = namespace) val containerInfo = KubeContainerInfo("img-name", "pull-policy") - val envMap: Map = mapOf() + val orchestratorEnvSingleton: OrchestratorEnvSingleton = mockk() + every { orchestratorEnvSingleton.orchestratorEnvVars(any()) } returns emptyList() + every { orchestratorEnvSingleton.orchestratorEnvMap(any()) } returns emptyMap() val awsAssumedRoleEnv: List = listOf() val replSelectors = mapOf("test-selector" to "normal-repl") val replCustomSelectors = mapOf("test-selector" to "custom-repl") @@ -69,9 +72,9 @@ class PayloadKubeInputMapperTest { serializer, labeler, podNameGenerator, + orchestratorEnvSingleton, namespace, containerInfo, - envMap, awsAssumedRoleEnv, replConfigs, checkConfigs, @@ -150,7 +153,9 @@ class PayloadKubeInputMapperTest { val podNameGenerator: PodNameGenerator = mockk() every { podNameGenerator.getCheckPodName(any(), any(), any()) } returns podName val orchestratorContainerInfo = KubeContainerInfo("img-name", "pull-policy") - val orchestratorEnvMap: Map = mapOf() + val orchestratorEnvSingleton: OrchestratorEnvSingleton = mockk() + every { orchestratorEnvSingleton.orchestratorEnvVars(any()) } returns emptyList() + every { orchestratorEnvSingleton.orchestratorEnvMap(any()) } returns emptyMap() val awsAssumedRoleEnv: List = listOf(EnvVar("aws-assumed-role", "value", null)) val checkSelectors = mapOf("test-selector" to "normal-check") val pullPolicy = "pull-policy" @@ -171,9 +176,9 @@ class PayloadKubeInputMapperTest { serializer, labeler, podNameGenerator, + orchestratorEnvSingleton, namespace, orchestratorContainerInfo, - orchestratorEnvMap, awsAssumedRoleEnv, replConfigs, checkConfigs, @@ -268,7 +273,9 @@ class PayloadKubeInputMapperTest { val podNameGenerator: PodNameGenerator = mockk() every { podNameGenerator.getDiscoverPodName(any(), any(), any()) } returns podName val orchestratorContainerInfo = KubeContainerInfo("img-name", "pull-policy") - val orchestratorEnvMap: Map = mapOf() + val orchestratorEnvSingleton: OrchestratorEnvSingleton = mockk() + every { orchestratorEnvSingleton.orchestratorEnvVars(any()) } returns emptyList() + every { orchestratorEnvSingleton.orchestratorEnvMap(any()) } returns emptyMap() val awsAssumedRoleEnv: List = listOf(EnvVar("aws-assumed-role", "value", null)) val checkSelectors = mapOf("test-selector" to "normal-check") val pullPolicy = "pull-policy" @@ -289,9 +296,9 @@ class PayloadKubeInputMapperTest { serializer, labeler, podNameGenerator, + orchestratorEnvSingleton, namespace, orchestratorContainerInfo, - orchestratorEnvMap, awsAssumedRoleEnv, replConfigs, checkConfigs, @@ -377,7 +384,9 @@ class PayloadKubeInputMapperTest { val podNameGenerator: PodNameGenerator = mockk() every { podNameGenerator.getSpecPodName(any(), any(), any()) } returns podName val orchestratorContainerInfo = KubeContainerInfo("img-name", "pull-policy") - val orchestratorEnvMap: Map = mapOf() + val orchestratorEnvSingleton: OrchestratorEnvSingleton = mockk() + every { orchestratorEnvSingleton.orchestratorEnvVars(any()) } returns emptyList() + every { orchestratorEnvSingleton.orchestratorEnvMap(any()) } returns emptyMap() val awsAssumedRoleEnv: List = listOf(EnvVar("aws-assumed-role", "value", null)) val checkSelectors = mapOf("test-selector" to "normal-check") val pullPolicy = "pull-policy" @@ -396,9 +405,9 @@ class PayloadKubeInputMapperTest { serializer, labeler, podNameGenerator, + orchestratorEnvSingleton, namespace, orchestratorContainerInfo, - orchestratorEnvMap, awsAssumedRoleEnv, replConfigs, checkConfigs, diff --git a/build.gradle b/build.gradle index 4a4c7b076d4..fa08b775cdc 100644 --- a/build.gradle +++ b/build.gradle @@ -25,11 +25,11 @@ buildscript { plugins { id "base" id "com.dorongold.task-tree" version "2.1.1" - id "io.airbyte.gradle.jvm" version "0.28.0" apply false - id "io.airbyte.gradle.jvm.app" version "0.28.0" apply false - id "io.airbyte.gradle.jvm.lib" version "0.28.0" apply false - id "io.airbyte.gradle.docker" version "0.28.0" apply false - id "io.airbyte.gradle.publish" version "0.28.0" apply false + id "io.airbyte.gradle.jvm" version "0.31.0" apply false + id "io.airbyte.gradle.jvm.app" version "0.31.0" apply false + id "io.airbyte.gradle.jvm.lib" version "0.31.0" apply false + id "io.airbyte.gradle.docker" version "0.31.0" apply false + id "io.airbyte.gradle.publish" version "0.31.0" apply false // uncomment for testing plugin locally // id "io.airbyte.gradle.jvm" version "local-test" apply false // id "io.airbyte.gradle.jvm.app" version "local-test" apply false @@ -158,17 +158,15 @@ subprojects { sp -> def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64' def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64' def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14' - def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:alpine' : 'amd64/nginx:alpine' // Used by the platform -- Must be an Amazon Corretto-based image for build to work without modification to Dockerfile instructions - def openjdkImage = System.getenv('JDK_IMAGE') ?: 'airbyte/airbyte-base-java-image:3.0.1' + def openjdkImage = System.getenv('JDK_IMAGE') ?: 'airbyte/airbyte-base-java-image:3.1.0' platform = buildPlatform images.add("airbyte/$sp.dockerImageName:$rootProject.ext.image_tag") buildArgs.put('JDK_VERSION', jdkVersion) buildArgs.put('DOCKER_BUILD_ARCH', buildArch) buildArgs.put('ALPINE_IMAGE', alpineImage) - buildArgs.put('NGINX_IMAGE', nginxImage) buildArgs.put('JDK_IMAGE', openjdkImage) buildArgs.put('VERSION', rootProject.ext.version) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index c17d7b12a11..1cc2dacca19 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index b4e1a8a2654..4b76dad5617 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 63922eae518..c2cfd42282b 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-connector-builder-server/templates/deployment.yaml b/charts/airbyte-connector-builder-server/templates/deployment.yaml index 74275642a83..cc715f17b2e 100644 --- a/charts/airbyte-connector-builder-server/templates/deployment.yaml +++ b/charts/airbyte-connector-builder-server/templates/deployment.yaml @@ -146,7 +146,7 @@ spec: ports: - name: http - containerPort: 80 + containerPort: 8080 protocol: TCP {{- if .Values.debug.enabled }} - name: debug diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index 9132e22827f..d168d6b76fb 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index c062cf4b6f6..add577dd482 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak-setup/templates/job.yaml b/charts/airbyte-keycloak-setup/templates/job.yaml index 7d408f4b9e3..77e1fad85c9 100644 --- a/charts/airbyte-keycloak-setup/templates/job.yaml +++ b/charts/airbyte-keycloak-setup/templates/job.yaml @@ -71,6 +71,21 @@ spec: configMapKeyRef: name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} key: KEYCLOAK_INTERNAL_HOST + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.database.secretName | default (printf "%s-airbyte-secrets" .Release.Name ) }} + key: {{ .Values.global.database.secretValue | default "DATABASE_PASSWORD" }} + - name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_URL + - name: DATABASE_USER + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-airbyte-secrets + key: DATABASE_USER # Values from secret {{- if .Values.secrets }} {{- range $k, $v := .Values.secrets }} diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index d876cbf293a..dc5823124e8 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 7b539d17fb8..f3b34e56082 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 252454f946f..b943dfb57e5 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 5950286935d..985a2d38621 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-server/templates/deployment.yaml b/charts/airbyte-server/templates/deployment.yaml index ab4427e1a11..bc439e0ce47 100644 --- a/charts/airbyte-server/templates/deployment.yaml +++ b/charts/airbyte-server/templates/deployment.yaml @@ -67,6 +67,11 @@ spec: value: "true" {{- end }} {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_API_HOST - name: AIRBYTE_VERSION valueFrom: configMapKeyRef: diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index f68b61a434e..974a88d4fb2 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 5045e45156f..db1d821d2cf 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/templates/deployment.yaml b/charts/airbyte-webapp/templates/deployment.yaml index 881796426a9..a0f21e42000 100644 --- a/charts/airbyte-webapp/templates/deployment.yaml +++ b/charts/airbyte-webapp/templates/deployment.yaml @@ -141,7 +141,7 @@ spec: {{- end }} ports: - name: http - containerPort: 80 + containerPort: 8080 protocol: TCP {{- if .Values.resources }} resources: {{- toYaml .Values.resources | nindent 10 }} diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 005c49bed06..ae3565ffabf 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/templates/deployment.yaml b/charts/airbyte-worker/templates/deployment.yaml index 16938e791fe..b501ff6c804 100644 --- a/charts/airbyte-worker/templates/deployment.yaml +++ b/charts/airbyte-worker/templates/deployment.yaml @@ -71,6 +71,12 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env key: CONFIG_ROOT + {{- if eq (lower (default "" .Values.global.storage.type)) "gcs" }} + - name: CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH + value: "/secrets/gcs-log-creds" + - name: CONTAINER_ORCHESTRATOR_SECRET_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }} + {{- end }} - name: DATABASE_HOST valueFrom: configMapKeyRef: diff --git a/charts/airbyte-workload-api-server/Chart.yaml b/charts/airbyte-workload-api-server/Chart.yaml index 15159bdbcb0..f7c2d8cbc29 100644 --- a/charts/airbyte-workload-api-server/Chart.yaml +++ b/charts/airbyte-workload-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-workload-launcher/Chart.yaml b/charts/airbyte-workload-launcher/Chart.yaml index 8b924b25da3..f4509e815e0 100644 --- a/charts/airbyte-workload-launcher/Chart.yaml +++ b/charts/airbyte-workload-launcher/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-workload-launcher/templates/deployment.yaml b/charts/airbyte-workload-launcher/templates/deployment.yaml index 96c101ab062..9a4aa5884ae 100644 --- a/charts/airbyte-workload-launcher/templates/deployment.yaml +++ b/charts/airbyte-workload-launcher/templates/deployment.yaml @@ -15,6 +15,8 @@ spec: {{- if .Values.extraSelectorLabels }} {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} {{- end }} + strategy: + type: Recreate template: metadata: labels: @@ -73,6 +75,12 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env key: CONFIG_ROOT + {{- if eq (lower (default "" .Values.global.storage.type)) "gcs" }} + - name: CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH + value: "/secrets/gcs-log-creds" + - name: CONTAINER_ORCHESTRATOR_SECRET_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }} + {{- end }} - name: DATABASE_HOST valueFrom: configMapKeyRef: @@ -98,6 +106,11 @@ spec: secretKeyRef: name: {{ .Release.Name }}-airbyte-secrets key: DATABASE_USER + - name: LOG4J_CONFIGURATION_FILE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: LOG4J_CONFIGURATION_FILE - name: MICROMETER_METRICS_ENABLED valueFrom: configMapKeyRef: diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 6ceb9d8888a..4d4e5248c3d 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,45 +4,45 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: workload-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: workload-launcher repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 -digest: sha256:d09c4b19b6abcef51e292a779de2dc1facb182bc517a42ff4fc761609f9959c2 -generated: "2024-03-14T00:09:18.181413097Z" + version: 0.60.8 +digest: sha256:4cc4b44abc523b5d7dae3c0ebd12d2f0085e0e11329d51750f1f65c25aafecff +generated: "2024-03-26T00:36:56.435009119Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 57ac3a7fc42..fa7218e2ef8 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.55.40 +version: 0.60.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,56 +32,56 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: workload-api-server.enabled name: workload-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: workload-launcher.enabled name: workload-launcher repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.55.40 + version: 0.60.8 diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index 7843490d71b..1370a0ce7ec 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -22,12 +22,12 @@ data: CONFIG_ROOT: /configs CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.35.15.001" DATA_DOCKER_MOUNT: airbyte_data + DB_DOCKER_MOUNT: airbyte_db DATABASE_DB: {{ include "airbyte.database.name" . }} DATABASE_HOST: {{ include "airbyte.database.host" . }} # todo: database information into secrets DATABASE_PORT: {{ include "airbyte.database.port" . | quote }} DATABASE_URL: {{ include "airbyte.database.url" . | quote }} KEYCLOAK_DATABASE_URL: {{ include "keycloak.database.url" . | quote }} - DB_DOCKER_MOUNT: airbyte_db GOOGLE_APPLICATION_CREDENTIALS: {{ include "airbyte.gcpLogCredentialsPath" . | quote }} INTERNAL_API_HOST: {{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }} {{- if eq (index .Values "workload-api-server" "enabled") true }} @@ -46,7 +46,11 @@ data: KEYCLOAK_INTERNAL_HOST: localhost # just a placeholder so that nginx template is valid - shouldn't be used when edition isn't "pro" {{- end }} CONNECTOR_BUILDER_API_HOST: {{ .Release.Name }}-airbyte-connector-builder-server-svc:{{ index .Values "connector-builder-server" "service" "port" }} - AIRBYTE_API_HOST: {{ .Release.Name }}-airbyte-api-server-svc:{{ index .Values "airbyte-api-server" "service" "port" }} +{{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + AIRBYTE_API_HOST: {{ printf "%s/api/public" (index $airbyteYmlDict "webapp-url") | quote }} +{{- else }} + AIRBYTE_API_HOST: http://{{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }}/api/public +{{- end }} {{- if $.Values.global.jobs.kube.annotations }} JOB_KUBE_ANNOTATIONS: {{ $.Values.global.jobs.kube.annotations | include "airbyte.flattenMap" | quote }} {{- end }} diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 8c60ee8d102..4a898340294 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -127,12 +127,12 @@ webapp: podLabels: {} # -- Security context for the container - ## Examples: - ## containerSecurityContext: - ## runAsNonRoot: true - ## runAsUser: 1000 - ## readOnlyRootFilesystem: true - containerSecurityContext: {} + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + readOnlyRootFilesystem: false ## Configure extra options for the webapp containers' liveness and readiness probes, ## see https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes @@ -342,13 +342,13 @@ pod-sweeper: # -- Add extra labels to the podSweeper pod podLabels: {} - ## Examples: - ## containerSecurityContext: - ## runAsNonRoot: true - ## runAsUser: 1000 - ## readOnlyRootFilesystem: true # -- Security context for the container - containerSecurityContext: {} + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + readOnlyRootFilesystem: false livenessProbe: # -- Enable livenessProbe on the podSweeper @@ -440,13 +440,13 @@ server: # -- Add extra labels to the server pods podLabels: {} - ## Examples: - ## containerSecurityContext: - ## runAsNonRoot: true - ## runAsUser: 1000 - ## readOnlyRootFilesystem: true # -- Security context for the container - containerSecurityContext: {} + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false ## Configure extra options for the server containers' liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes @@ -612,13 +612,13 @@ worker: # -- Add extra labels to the worker pods podLabels: {} - ## Examples: - ## containerSecurityContext: - ## runAsNonRoot: true - ## runAsUser: 1000 - ## readOnlyRootFilesystem: true # -- Security context for the container - containerSecurityContext: {} + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false livenessProbe: # -- Enable livenessProbe on the worker @@ -741,13 +741,13 @@ workload-launcher: # -- Add extra labels to the workload launcher pods podLabels: {} - ## Examples: - ## containerSecurityContext: - ## runAsNonRoot: true - ## runAsUser: 1000 - ## readOnlyRootFilesystem: true # -- Security context for the container - containerSecurityContext: {} + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false livenessProbe: # -- Enable livenessProbe on the workload launcher @@ -873,13 +873,13 @@ metrics: # -- Add extra labels to the metrics-reporter pod podLabels: {} - ## Examples: - ## containerSecurityContext: - ## runAsNonRoot: true - ## runAsUser: 1000 - ## readOnlyRootFilesystem: true # -- Security context for the container - containerSecurityContext: {} + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false ## metrics-reporter app resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ @@ -985,6 +985,14 @@ airbyte-bootloader: # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity affinity: {} + # -- Security context for the container + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + ## Example: (With default env vars and values taken from generated config map) ## extraEnv: ## - name: AIRBYTE_VERSION @@ -1098,13 +1106,13 @@ temporal: # -- Add extra labels to the temporal pod podLabels: {} - ## Examples: - ## containerSecurityContext: - ## runAsNonRoot: true - ## runAsUser: 1000 - ## readOnlyRootFilesystem: true # -- Security context for the container - containerSecurityContext: {} + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): ## extraInitContainers: @@ -1286,13 +1294,13 @@ cron: # -- Add extra labels to the cron pods podLabels: {} - ## Examples: - ## containerSecurityContext: - ## runAsNonRoot: true - ## runAsUser: 1000 - ## readOnlyRootFilesystem: true # -- Security context for the container - containerSecurityContext: {} + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false livenessProbe: # -- Enable livenessProbe on the cron @@ -1447,6 +1455,14 @@ connector-builder-server: # -- The pull policy to use for the airbyte connector-builder-server image pullPolicy: IfNotPresent + # -- Security context for the container + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + livenessProbe: # -- Enable livenessProbe on the server enabled: true @@ -1514,6 +1530,14 @@ airbyte-api-server: # -- The pull policy to use for the airbyte airbyte-api-server image pullPolicy: IfNotPresent + # -- Security context for the container + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + livenessProbe: # -- Enable livenessProbe on the server enabled: true @@ -1600,10 +1624,25 @@ keycloak: adminUsername: airbyteAdmin adminPassword: keycloak123 + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + keycloak-setup: enabled: true env_vars: {} + # -- Security context for the container + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + workload-api-server: enabled: false @@ -1618,6 +1657,14 @@ workload-api-server: # -- The pull policy to use for the airbyte-workload-api-server image pullPolicy: IfNotPresent + # -- Security context for the container + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + livenessProbe: # -- Enable livenessProbe on the server enabled: true diff --git a/deps.toml b/deps.toml index f0f676faed8..0339c15a6f4 100644 --- a/deps.toml +++ b/deps.toml @@ -1,5 +1,5 @@ [versions] -airbyte-protocol = "0.5.1" +airbyte-protocol = "0.8.0" bouncycastle = "1.70" commons_io = "2.7" connectors-testcontainers = "1.15.3" @@ -20,7 +20,7 @@ kotlin-logging = "5.1.0" kubernetes-client = "6.5.1" log4j = "2.22.1" lombok = "1.18.30" -micronaut = "4.3.5" +micronaut = "4.3.7" micronaut-cache = "4.2.2" micronaut-data = "4.6.1" micronaut-email = "2.4.0" @@ -29,7 +29,7 @@ micronaut-jdbc = "5.5.1" micronaut-kotlin = "4.2.0" micronaut-micrometer = "5.4.0" micronaut-openapi = "6.5.1" -micronaut-security = "4.6.7" +micronaut-security = "4.6.9" micronaut-test = "4.2.0" moshi = "1.15.0" mockito = "5.8.0" @@ -162,8 +162,8 @@ quartz-scheduler = { module = "org.quartz-scheduler:quartz", version = "2.3.2" } reactor-core = { module = "io.projectreactor:reactor-core", version.ref = "reactor" } reactor-kotlin-extensions = { module = "io.projectreactor.kotlin:reactor-kotlin-extensions", version = "1.2.2" } reactor-test = { module = "io.projectreactor:reactor-test", version.ref = "reactor" } -s3 = { module = "software.amazon.awssdk:s3", version = "2.16.84" } -sts = { module = "software.amazon.awssdk:sts", version = "2.20.162" } +s3 = { module = "software.amazon.awssdk:s3", version = "2.23.17" } +sts = { module = "software.amazon.awssdk:sts", version = "2.23.17" } segment-java-analytics = { module = "com.segment.analytics.java:analytics", version.ref = "segment" } sendgrid-java = { module = "com.sendgrid:sendgrid-java", version = "4.0.1" } sentry-java = { module = "io.sentry:sentry", version.ref = "sentry" } @@ -224,7 +224,7 @@ micronaut-runtime = { module = "io.micronaut:micronaut-runtime", version.ref = " micronaut-security = { module = "io.micronaut.security:micronaut-security", version.ref = "micronaut-security" } micronaut-test-core = { module = "io.micronaut.test:micronaut-test-core", version.ref = "micronaut-test" } micronaut-test-junit5 = { module = "io.micronaut.test:micronaut-test-junit5", version.ref = "micronaut-test" } -micronaut-validation = { module = "io.micronaut.validation:micronaut-validation", version = "4.4.3" } +micronaut-validation = { module = "io.micronaut.validation:micronaut-validation", version = "4.4.4" } [bundles] apache = ["apache-commons", "apache-commons-lang"] @@ -257,5 +257,5 @@ temporal-telemetry = ["temporal-opentracing"] [plugins] ksp = { id ="com.google.devtools.ksp", version = "1.9.22-1.0.17"} node-gradle = { id = "com.github.node-gradle.node", version = "3.4.0" } -nu-studer-jooq = { id = "nu.studer.jooq", version = "8.2.1" } +nu-studer-jooq = { id = "nu.studer.jooq", version = "9.0" } de-undercouch-download = { id = "de.undercouch.download", version = "5.5.0" } diff --git a/docker-compose.acceptance-test.yaml b/docker-compose.acceptance-test.yaml index c052d9fe3cc..fdac2cbbe19 100644 --- a/docker-compose.acceptance-test.yaml +++ b/docker-compose.acceptance-test.yaml @@ -37,7 +37,7 @@ services: # written requires the webapp to run. So we pin the webapp to a specific commit. # This is not expected to change frequently - or at all. webapp: - image: "airbyte/webapp:dev-0e74a851e8" + image: "airbyte/webapp:dev-521ad4842c" configs: flags: file: ./tools/bin/acceptance-test-flags.yml diff --git a/docker-compose.workloads.yaml b/docker-compose.workloads.yaml index 26e5d982d9c..4dd18e89970 100644 --- a/docker-compose.workloads.yaml +++ b/docker-compose.workloads.yaml @@ -51,7 +51,7 @@ services: volumes: - workspace:${WORKSPACE_ROOT} - data:${CONFIG_ROOT} - - ${LOCAL_ROOT}:${LOCAL_ROOT} + - local_root:${LOCAL_ROOT} - ./configs:/app/configs:ro networks: - airbyte_internal @@ -73,6 +73,7 @@ services: - DATABASE_URL=${DATABASE_URL} - DATABASE_USER=${DATABASE_USER} - DATA_PLANE_ID=local + - DOCKER_HOST=docker-proxy:2375 - DOCKER_NETWORK=${COMPOSE_PROJECT_NAME}_airbyte_internal - DOCKER_SOCKET=${DOCKER_SOCKET} - DEPLOYMENT_MODE=${DEPLOYMENT_MODE} @@ -108,10 +109,9 @@ services: configs: - flags volumes: - - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock - workspace:${WORKSPACE_ROOT} - data:${CONFIG_ROOT} - - ${LOCAL_ROOT}:${LOCAL_ROOT} + - local_root:${LOCAL_ROOT} - ./configs:/app/configs:ro networks: - airbyte_internal diff --git a/docker-compose.yaml b/docker-compose.yaml index b1115a1bb7f..5e7e40c5f6b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,6 +6,16 @@ x-logging: &default-logging max-file: "5" driver: json-file services: + docker-proxy: + image: alpine/socat + command: -d -d -t 60 TCP-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock + ports: + - "2375" + user: root + volumes: + - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock + networks: + - airbyte_internal # hook in case we need to add init behavior # every root service (no depends_on) should depend on init init: @@ -80,6 +90,7 @@ services: - DEPLOYMENT_MODE=${DEPLOYMENT_MODE} - DD_AGENT_HOST=${DD_AGENT_HOST} - DD_DOGSTATSD_PORT=${DD_DOGSTATSD_PORT} + - DOCKER_HOST=docker-proxy:2375 - FEATURE_FLAG_CLIENT=${FEATURE_FLAG_CLIENT} - FIELD_SELECTION_WORKSPACES=${FIELD_SELECTION_WORKSPACES} - INTERNAL_API_HOST=${INTERNAL_API_HOST} @@ -92,7 +103,7 @@ services: - JOB_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_MAIN_CONTAINER_MEMORY_REQUEST} - LAUNCHDARKLY_KEY=${LAUNCHDARKLY_KEY} - LOCAL_DOCKER_MOUNT=${LOCAL_DOCKER_MOUNT} - - LOCAL_ROOT=${LOCAL_DOCKER_MOUNT} + - LOCAL_ROOT=${LOCAL_ROOT} - LOG_CONNECTOR_MESSAGES=${LOG_CONNECTOR_MESSAGES} - LOG_LEVEL=${LOG_LEVEL} - MAX_CHECK_WORKERS=${MAX_CHECK_WORKERS} @@ -132,9 +143,8 @@ services: configs: - flags volumes: - - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock - workspace:${WORKSPACE_ROOT} - - ${LOCAL_ROOT}:${LOCAL_ROOT} + - local_root:${LOCAL_ROOT} ports: - "9000" networks: @@ -202,7 +212,7 @@ services: volumes: - workspace:${WORKSPACE_ROOT} - data:${CONFIG_ROOT} - - ${LOCAL_ROOT}:${LOCAL_ROOT} + - local_root:${LOCAL_ROOT} - ./configs:/app/configs:ro networks: - airbyte_internal @@ -308,7 +318,7 @@ services: container_name: airbyte-connector-builder-server restart: unless-stopped ports: - - 80 + - 8080 environment: - AIRBYTE_VERSION=${VERSION} - CDK_VERSION=${CDK_VERSION} @@ -349,6 +359,8 @@ services: volumes: workspace: name: ${WORKSPACE_DOCKER_MOUNT} + local_root: + name: ${LOCAL_DOCKER_MOUNT} # the data volume is only needed for backward compatibility; when users upgrade # from an old Airbyte version that relies on file-based configs, the server needs # to read this volume to copy their configs to the database diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 00000000000..f4306afae7d --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,57 @@ +##@ Images: + +VERSION ?= $(shell git rev-parse --short HEAD) +IMAGES_DIR := ./oss/docker +OS ?= $(shell uname | tr '[:upper:]' '[:lower:]') + +PUBLISH ?= false + +BASE_JAVA_IMAGE_VERSION ?= 3.1.0 +BASE_JAVA_IMAGE = airbyte/airbyte-base-java-image:$(BASE_JAVA_IMAGE_VERSION) + +image.base-images: ## Build all the base images +image.base-images: buildx.start +image.base-images: image.airbyte-base-java-image image.airbyte-base-java-python-image image.airbyte-base-java-worker-image + +image.airbyte-base-java-image: ## Build the airbyte-base-java-image +image.airbyte-base-java-image: buildx.start + @if [ "$(PUBLISH)" = "true" ]; then \ + docker buildx build -t airbyte/airbyte-base-java-image:$(VERSION) \ + --builder airbyte-image-builder \ + --platform linux/amd64,linux/arm64 \ + --push \ + -f $(IMAGES_DIR)/airbyte-base-java-image/Dockerfile . ; \ + else \ + docker build -t airbyte/airbyte-base-java-image:$(VERSION) -f $(IMAGES_DIR)/airbyte-base-java-image/Dockerfile . ; \ + fi + +image.airbyte-base-java-python-image: ## Build the airbyte-base-java-python-image +image.airbyte-base-java-python-image: buildx.start + @if [ "$(PUBLISH)" = "true" ]; then \ + docker buildx build -t airbyte/airbyte-base-java-python-image:$(VERSION) \ + --builder airbyte-image-builder \ + --build-arg JDK_IMAGE=$(BASE_JAVA_IMAGE) \ + --platform linux/amd64,linux/arm64 \ + --push \ + -f $(IMAGES_DIR)/airbyte-base-java-python-image/Dockerfile . ; \ + else \ + docker build \ + --build-arg JDK_IMAGE=$(BASE_JAVA_IMAGE) \ + -t airbyte/airbyte-base-java-python-image:$(VERSION) \ + -f $(IMAGES_DIR)/airbyte-base-java-python-image/Dockerfile . ; \ + fi + +image.airbyte-base-java-worker-image: ## Build the airbyte-base-java-worker-image + @if [ "$(PUBLISH)" = "true" ]; then \ + docker buildx build -t airbyte/airbyte-base-java-worker-image:$(VERSION) \ + --build-arg JDK_IMAGE=$(BASE_JAVA_IMAGE) \ + --platform linux/amd64,linux/arm64 \ + --push \ + -f $(IMAGES_DIR)/airbyte-base-java-worker-image/Dockerfile . ; \ + else \ + docker build \ + --build-arg JDK_IMAGE=$(BASE_JAVA_IMAGE) \ + -t airbyte/airbyte-base-java-worker-image:$(VERSION) \ + -f $(IMAGES_DIR)/airbyte-base-java-worker-image/Dockerfile . ; \ + fi + \ No newline at end of file diff --git a/airbyte-base-java-image/Dockerfile b/docker/airbyte-base-java-image/Dockerfile similarity index 54% rename from airbyte-base-java-image/Dockerfile rename to docker/airbyte-base-java-image/Dockerfile index c51bbd06982..7c72e0b8113 100644 --- a/airbyte-base-java-image/Dockerfile +++ b/docker/airbyte-base-java-image/Dockerfile @@ -2,12 +2,22 @@ FROM amazoncorretto:21 ARG DOCKER_BUILD_ARCH=amd64 -WORKDIR /app +RUN yum install -y tar shadow-utils + +RUN groupadd --gid 1000 airbyte \ + && useradd --uid 1000 --gid airbyte --shell /bin/bash --create-home airbyte -RUN yum install -y tar +WORKDIR /app # Add the Datadog Java APM agent ADD https://dtdg.co/latest-java-tracer dd-java-agent.jar # Add the OpenTelemetry Java APM agent ADD https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar opentelemetry-javaagent.jar + +# Create mount point for secrets +RUN mkdir /secrets && chown -R airbyte:airbyte /secrets + +RUN chown -R airbyte:airbyte /app + +USER airbyte:airbyte diff --git a/airbyte-base-java-image/README.md b/docker/airbyte-base-java-image/README.md similarity index 95% rename from airbyte-base-java-image/README.md rename to docker/airbyte-base-java-image/README.md index 3a3b5c27e19..405b7b4597e 100644 --- a/airbyte-base-java-image/README.md +++ b/docker/airbyte-base-java-image/README.md @@ -12,7 +12,7 @@ To release a new version of this base image, use the following steps: 3. Run the following to build and push a new version of this image (replace `` with a new version!) : ``` docker buildx build --push \ - --tag airbyte/airbyte-base-java-image: \ + --tag airbyte/airbyte-base-java-worker-image: \ --platform linux/amd64,linux/arm64 . ``` To see existing versions, [view the image on Dockerhub](https://hub.docker.com/r/airbyte/airbyte-base-java-image). diff --git a/docker/airbyte-base-java-python-image/Dockerfile b/docker/airbyte-base-java-python-image/Dockerfile new file mode 100644 index 00000000000..9ef7dcc4558 --- /dev/null +++ b/docker/airbyte-base-java-python-image/Dockerfile @@ -0,0 +1,18 @@ +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.1.0 +FROM ${JDK_IMAGE} + +USER root + +RUN yum update -y && \ + yum groupinstall -y "Development Tools" && \ + yum install -y gcc make patch zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl11-devel tk-devel libffi-devel xz-devel + +USER airbyte:airbyte + +ENV PYTHON_VERSION=3.9.11 + +# Set up python +RUN git clone https://github.com/pyenv/pyenv.git ~/.pyenv +ENV PYENV_ROOT /home/airbyte/.pyenv +ENV PATH ${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:$PATH +RUN pyenv install -v ${PYTHON_VERSION} && pyenv global ${PYTHON_VERSION} diff --git a/airbyte-base-java-python-image/README.md b/docker/airbyte-base-java-python-image/README.md similarity index 100% rename from airbyte-base-java-python-image/README.md rename to docker/airbyte-base-java-python-image/README.md diff --git a/airbyte-base-java-worker-image/Dockerfile b/docker/airbyte-base-java-worker-image/Dockerfile similarity index 53% rename from airbyte-base-java-worker-image/Dockerfile rename to docker/airbyte-base-java-worker-image/Dockerfile index a387018db72..82d5a889490 100644 --- a/airbyte-base-java-worker-image/Dockerfile +++ b/docker/airbyte-base-java-worker-image/Dockerfile @@ -1,9 +1,17 @@ -FROM airbyte/airbyte-base-java-image:3.0.1 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.1.0 +FROM ${JDK_IMAGE} ARG TARGETPLATFORM +USER root + RUN amazon-linux-extras install -y docker RUN yum install -y jq tar && yum clean all RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/$TARGETPLATFORM/kubectl" \ && chmod +x kubectl && mv kubectl /usr/local/bin/ + +RUN mkdir -p /tmp/workspace && chown -R airbyte:airbyte /tmp/workspace +RUN mkdir -p /tmp/airbyte_local && chown -R airbyte:airbyte /tmp/airbyte_local + +USER airbyte:airbyte diff --git a/airbyte-base-java-worker-image/README.md b/docker/airbyte-base-java-worker-image/README.md similarity index 100% rename from airbyte-base-java-worker-image/README.md rename to docker/airbyte-base-java-worker-image/README.md diff --git a/settings.gradle.kts b/settings.gradle.kts index cd2372f686b..34e2d39ede7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,74 +1,74 @@ pluginManagement { - repositories { - // uncomment for local dev - // maven { - // name = "localPluginRepo" - // url = uri("../.gradle-plugins-local") - // } - maven(url = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars") - gradlePluginPortal() - maven(url = "https://oss.sonatype.org/content/repositories/snapshots") - } - resolutionStrategy { - eachPlugin { - // We're using the 6.1.0-SNAPSHOT version of openapi-generator which contains a fix for generating nullable arrays (https://github.com/OpenAPITools/openapi-generator/issues/13025) - // The snapshot version isn"t available in the main Gradle Plugin Portal, so we added the Sonatype snapshot repository above. - // The useModule command below allows us to map from the plugin id, `org.openapi.generator`, to the underlying module (https://oss.sonatype.org/content/repositories/snapshots/org/openapitools/openapi-generator-gradle-plugin/6.1.0-SNAPSHOT/_ - if (requested.id.id == "org.openapi.generator") { - useModule("org.openapitools:openapi-generator-gradle-plugin:${requested.version}") - } - } + repositories { + // uncomment for local dev + // maven { + // name = "localPluginRepo" + // url = uri("../.gradle-plugins-local") + // } + maven(url = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars") + gradlePluginPortal() + maven(url = "https://oss.sonatype.org/content/repositories/snapshots") + } + resolutionStrategy { + eachPlugin { + // We're using the 6.1.0-SNAPSHOT version of openapi-generator which contains a fix for generating nullable arrays (https://github.com/OpenAPITools/openapi-generator/issues/13025) + // The snapshot version isn"t available in the main Gradle Plugin Portal, so we added the Sonatype snapshot repository above. + // The useModule command below allows us to map from the plugin id, `org.openapi.generator`, to the underlying module (https://oss.sonatype.org/content/repositories/snapshots/org/openapitools/openapi-generator-gradle-plugin/6.1.0-SNAPSHOT/_ + if (requested.id.id == "org.openapi.generator") { + useModule("org.openapitools:openapi-generator-gradle-plugin:${requested.version}") + } } + } } // Configure the gradle enterprise plugin to enable build scans. Enabling the plugin at the top of the settings file allows the build scan to record // as much information as possible. plugins { - id("com.gradle.enterprise") version "3.15.1" - id("com.github.burrunan.s3-build-cache") version "1.5" + id("com.gradle.enterprise") version "3.15.1" + id("com.github.burrunan.s3-build-cache") version "1.5" } gradleEnterprise { - buildScan { - termsOfServiceUrl = "https://gradle.com/terms-of-service" - termsOfServiceAgree = "yes" - } + buildScan { + termsOfServiceUrl = "https://gradle.com/terms-of-service" + termsOfServiceAgree = "yes" + } } val isCiServer = System.getenv().containsKey("CI") gradleEnterprise { - buildScan { - isUploadInBackground = !isCiServer // Disable async upload so that the containers doesn't terminate the upload - buildScanPublished { - file("scan-journal.log").writeText("${java.util.Date()} - $buildScanId - ${buildScanUri}\n") - } + buildScan { + isUploadInBackground = !isCiServer // Disable async upload so that the containers doesn't terminate the upload + buildScanPublished { + file("scan-journal.log").writeText("${java.util.Date()} - $buildScanId - ${buildScanUri}\n") } + } } buildCache { - remote { - region = "us-west-2" - bucket = "ab-ci-cache" - prefix = "platform-ci-cache/" - isPush = isCiServer - isEnabled = System.getenv().containsKey("S3_BUILD_CACHE_ACCESS_KEY_ID") - } + remote { + region = "us-west-2" + bucket = "ab-ci-cache" + prefix = "platform-ci-cache/" + isPush = isCiServer + isEnabled = System.getenv().containsKey("S3_BUILD_CACHE_ACCESS_KEY_ID") + } } rootProject.name = "airbyte" // definition for dependency resolution dependencyResolutionManagement { - repositories { - maven(url = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/") - } + repositories { + maven(url = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/") + } - versionCatalogs { - create("libs") { - from(files("deps.toml")) - } + versionCatalogs { + create("libs") { + from(files("deps.toml")) } + } } // todo (cgardens) - alphabetize diff --git a/tools/bin/publish_docker.sh b/tools/bin/publish_docker.sh index b1c99f997e3..0763a0c32d1 100755 --- a/tools/bin/publish_docker.sh +++ b/tools/bin/publish_docker.sh @@ -16,6 +16,8 @@ projectDir=( "temporal" "webapp" "workers" + "workload-api-server" + "workload-launcher" "keycloak" "keycloak-setup" "api-server" @@ -23,8 +25,8 @@ projectDir=( # Set default values to required vars. If set in env, values will be taken from there. # Primarily for testing. -JDK_VERSION=${JDK_VERSION:-17.0.4} -ALPINE_IMAGE=${ALPINE_IMAGE:-alpine:3.14} +JDK_VERSION=${JDK_VERSION:-21.1.0} +ALPINE_IMAGE=${ALPINE_IMAGE:-alpine:3.18} POSTGRES_IMAGE=${POSTGRES_IMAGE:-postgres:13-alpine} # Iterate over all directories in list to build one by one.