Merge branch 'main' into minio-docker-pull-secret

.bumpversion.cfg

@@ -1,51 +1,82 @@
 [bumpversion]
-current_version = 0.50.23
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
 serialize = 
 	{major}.{minor}.{patch}
 
-[bumpversion:file:.bumpversion.cfg]
-
 [bumpversion:file:.env]
+search = VERSION=dev
+replace = VERSION={new_version}
 
 [bumpversion:file:airbyte-container-orchestrator/Dockerfile]
+search = ARG VERSION=dev
+replace = ARG VERSION={new_version}
 
 [bumpversion:file:airbyte-proxy/Dockerfile]
+search = ARG VERSION=dev
+replace = ARG VERSION={new_version}
 
 [bumpversion:file:airbyte-server/Dockerfile]
+search = ARG VERSION=dev
+replace = ARG VERSION={new_version}
 
-[bumpversion:file:airbyte-webapp/package.json]
-search = "version": "{current_version}"
-replace = "version": "{new_version}"
+[bumpversion:file:airbyte-connector-builder-server/Dockerfile]
+search = ARG VERSION=dev
+replace = ARG VERSION={new_version}
 
 [bumpversion:file:charts/airbyte-bootloader/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-cron/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-metrics/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-pod-sweeper/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-server/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-temporal/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-webapp/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-worker/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-connector-builder-server/Chart.yaml]
-
-[bumpversion:file:charts/airbyte/README.md]
-
-[bumpversion:file:airbyte-connector-builder-server/Dockerfile]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-keycloak/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-keycloak-setup/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
 
 [bumpversion:file:charts/airbyte-api-server/Chart.yaml]
+search = appVersion: dev
+replace = appVersion: {new_version}
+
+[bumpversion:file:charts/airbyte/README.md]
+search = ![appVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square)
+replace = ![appVersion: {new_version}](https://img.shields.io/badge/AppVersion-{new_version}-informational?style=flat-square)

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.50.23
+VERSION=dev
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.gitignore

@@ -63,9 +63,6 @@ crash.log
 resources/examples/airflow/logs/*
 !resources/examples/airflow/logs/.gitkeep
 
-# Cloud Demo
-!airbyte-webapp/src/packages/cloud/data
-
 # Ignore docs folder, since we're using it to temporarily copy files into on CI
 /docs
 
@@ -87,4 +84,4 @@ dd-java-agent.jar
 
 # Ignore airbyte.yml since this file contains user-provided information that is sensitive and should not be committed
 # See airbyte.sample.yml for an example of the expected file structure.
-airbyte.yml
+/configs/airbyte.yml

airbyte-analytics/src/main/java/io/airbyte/analytics/SegmentTrackingClient.java

@@ -57,6 +57,7 @@ public class SegmentTrackingClient implements TrackingClient {
   private static final String AIRBYTE_TRACKED_AT = "tracked_at";
   protected static final String UNKNOWN = "unknown";
   protected static final String AIRBYTE_DEPLOYMENT_ID = "deployment_id";
+  protected static final String AIRBYTE_DEPLOYMENT_MODE = "deployment_mode";
 
   // Analytics is threadsafe.
   private final Analytics analytics;
@@ -138,6 +139,7 @@ public void track(@Nullable final UUID workspaceId, final String action, final M
     mapCopy.put(AIRBYTE_VERSION_KEY, trackingIdentity.getAirbyteVersion().serialize());
     mapCopy.put(CUSTOMER_ID_KEY, trackingIdentity.getCustomerId());
     mapCopy.put(AIRBYTE_DEPLOYMENT_ID, deployment.getDeploymentId());
+    mapCopy.put(AIRBYTE_DEPLOYMENT_MODE, deployment.getDeploymentMode());
     mapCopy.put(AIRBYTE_TRACKED_AT, Instant.now().toString());
     if (!metadata.isEmpty()) {
       trackingIdentity.getEmail().ifPresent(email -> mapCopy.put("email", email));

airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java

@@ -6,6 +6,7 @@
 
 import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_ANALYTIC_SOURCE_HEADER;
 import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_DEPLOYMENT_ID;
+import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_DEPLOYMENT_MODE;
 import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_SOURCE;
 import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_VERSION_KEY;
 import static io.airbyte.analytics.SegmentTrackingClient.UNKNOWN;
@@ -119,7 +120,8 @@ void testTrack() {
         ImmutableMap.of(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize(),
             "user_id", IDENTITY.getCustomerId(),
             AIRBYTE_SOURCE, UNKNOWN,
-            AIRBYTE_DEPLOYMENT_ID, DEPLOYMENT.getDeploymentId());
+            AIRBYTE_DEPLOYMENT_ID, DEPLOYMENT.getDeploymentId(),
+            AIRBYTE_DEPLOYMENT_MODE, DEPLOYMENT.getDeploymentMode());
 
     segmentTrackingClient.track(WORKSPACE_ID, JUMP);
 
@@ -139,7 +141,8 @@ void testTrackWithMetadata() {
         "height", "80 meters",
         "user_id", IDENTITY.getCustomerId(),
         AIRBYTE_SOURCE, UNKNOWN,
-        AIRBYTE_DEPLOYMENT_ID, DEPLOYMENT.getDeploymentId());
+        AIRBYTE_DEPLOYMENT_ID, DEPLOYMENT.getDeploymentId(),
+        AIRBYTE_DEPLOYMENT_MODE, DEPLOYMENT.getDeploymentMode());
 
     segmentTrackingClient.track(WORKSPACE_ID, JUMP, metadata);
 

airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ApiTrackingHelpers.kt → airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/ApiTrackingHelpers.kt

@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
  */
-package io.airbyte.api.server.helpers
+package io.airbyte.api.server.apiTracking
 
 import io.airbyte.analytics.Deployment
 import io.airbyte.analytics.TrackingClientSingleton
@@ -31,23 +31,29 @@ private val USER_ID = "user_id"
 private val ENDPOINT = "endpoint"
 private val OPERATION = "operation"
 private val STATUS_CODE = "status_code"
-private val API_VERSION = "api_version"
 private val WORKSPACE = "workspace"
 
 fun setupTrackingClient(
-  airbyteVersion: AirbyteVersion?,
-  trackingStrategy: Configs.TrackingStrategy?,
+  airbyteVersion: AirbyteVersion,
+  deploymentMode: Configs.DeploymentMode,
+  trackingStrategy: Configs.TrackingStrategy,
+  workerEnvironment: Configs.WorkerEnvironment,
 ) {
+  log.debug("deployment mode: $deploymentMode")
+  log.debug("airbyte version: $airbyteVersion")
+  log.debug("tracking strategy: $trackingStrategy")
+  log.debug("worker environment: $workerEnvironment")
+
   // fake a deployment UUID until we want to have the public api server talking to the database
   // directly
   val deploymentId = UUID.randomUUID()
   log.info("setting up tracking client with deploymentId: $deploymentId")
   TrackingClientSingleton.initializeWithoutDatabase(
     trackingStrategy,
     Deployment(
-      Configs.DeploymentMode.OSS,
+      deploymentMode,
       deploymentId,
-      Configs.WorkerEnvironment.KUBERNETES,
+      workerEnvironment,
     ),
     airbyteVersion,
   )
@@ -58,15 +64,13 @@ fun track(
   endpointPath: String?,
   httpOperation: String?,
   httpStatusCode: Int,
-  apiVersion: String?,
   workspaceId: Optional<UUID>,
 ) {
   val payload = mutableMapOf(
     Pair(USER_ID, userId),
     Pair(ENDPOINT, endpointPath),
     Pair(OPERATION, httpOperation),
     Pair(STATUS_CODE, httpStatusCode),
-    Pair(API_VERSION, apiVersion),
   )
   if (workspaceId.isPresent) {
     payload[WORKSPACE] = workspaceId.get().toString()

airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingClient.kt

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+ */
+package io.airbyte.api.server.apiTracking
+
+import io.airbyte.config.Configs
+import io.airbyte.config.EnvConfigs
+import jakarta.inject.Singleton
+
+/**
+ * Used to track usage of the airbyte-api-server.
+ */
+@Singleton
+class TrackingClient {
+  val configs: Configs = EnvConfigs()
+
+  init {
+    setupTrackingClient(configs.airbyteVersion, configs.deploymentMode, configs.trackingStrategy, configs.workerEnvironment)
+  }
+}

airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/TrackingHelper.kt → airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingHelper.kt

@@ -2,7 +2,7 @@
  * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
  */
 
-package io.airbyte.api.server.helpers
+package io.airbyte.api.server.apiTracking
 
 import io.micronaut.http.HttpStatus
 import org.zalando.problem.AbstractThrowableProblem
@@ -16,15 +16,13 @@ import javax.ws.rs.core.Response
  * Todo: This should be a middleware through a micronaut annotation so that we do not need to add this wrapper functions around all our calls.
  */
 object TrackingHelper {
-  private val API_VERSION = System.getenv("AIRBYTE_VERSION") ?: "unknown"
   private fun trackSuccess(endpointPath: String, httpOperation: String, userId: UUID, workspaceId: Optional<UUID>) {
     val statusCode = Response.Status.OK.statusCode
     track(
       userId,
       endpointPath,
       httpOperation,
       statusCode,
-      API_VERSION,
       workspaceId,
     )
   }
@@ -62,7 +60,6 @@ object TrackingHelper {
         endpointPath,
         httpOperation,
         statusCode,
-        API_VERSION,
         Optional.empty(),
       )
     }

airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt

@@ -13,14 +13,14 @@ import io.airbyte.api.client.model.generated.AirbyteCatalog
 import io.airbyte.api.client.model.generated.AirbyteStreamAndConfiguration
 import io.airbyte.api.client.model.generated.DestinationSyncMode
 import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRead
+import io.airbyte.api.server.apiTracking.TrackingHelper
 import io.airbyte.api.server.constants.CONNECTIONS_PATH
 import io.airbyte.api.server.constants.CONNECTIONS_WITH_ID_PATH
 import io.airbyte.api.server.constants.DELETE
 import io.airbyte.api.server.constants.GET
 import io.airbyte.api.server.constants.POST
 import io.airbyte.api.server.constants.PUT
 import io.airbyte.api.server.helpers.AirbyteCatalogHelper
-import io.airbyte.api.server.helpers.TrackingHelper
 import io.airbyte.api.server.helpers.getLocalUserInfoIfNull
 import io.airbyte.api.server.services.ConnectionService
 import io.airbyte.api.server.services.DestinationService

airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt

@@ -9,6 +9,7 @@ import io.airbyte.airbyte_api.generated.DestinationsApi
 import io.airbyte.airbyte_api.model.generated.DestinationCreateRequest
 import io.airbyte.airbyte_api.model.generated.DestinationPatchRequest
 import io.airbyte.airbyte_api.model.generated.DestinationPutRequest
+import io.airbyte.api.server.apiTracking.TrackingHelper
 import io.airbyte.api.server.constants.DELETE
 import io.airbyte.api.server.constants.DESTINATIONS_PATH
 import io.airbyte.api.server.constants.DESTINATIONS_WITH_ID_PATH
@@ -17,7 +18,6 @@ import io.airbyte.api.server.constants.GET
 import io.airbyte.api.server.constants.PATCH
 import io.airbyte.api.server.constants.POST
 import io.airbyte.api.server.constants.PUT
-import io.airbyte.api.server.helpers.TrackingHelper
 import io.airbyte.api.server.helpers.getIdFromName
 import io.airbyte.api.server.helpers.getLocalUserInfoIfNull
 import io.airbyte.api.server.helpers.removeDestinationType

airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt

@@ -11,13 +11,14 @@ import io.airbyte.airbyte_api.model.generated.JobStatusEnum
 import io.airbyte.airbyte_api.model.generated.JobTypeEnum
 import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody.OrderByFieldEnum
 import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody.OrderByMethodEnum
+import io.airbyte.api.server.apiTracking.TrackingHelper
 import io.airbyte.api.server.constants.DELETE
+import io.airbyte.api.server.constants.ENDPOINT_API_USER_INFO_HEADER
 import io.airbyte.api.server.constants.GET
 import io.airbyte.api.server.constants.JOBS_PATH
 import io.airbyte.api.server.constants.JOBS_WITH_ID_PATH
 import io.airbyte.api.server.constants.POST
 import io.airbyte.api.server.filters.JobsFilter
-import io.airbyte.api.server.helpers.TrackingHelper
 import io.airbyte.api.server.helpers.getLocalUserInfoIfNull
 import io.airbyte.api.server.problems.BadRequestProblem
 import io.airbyte.api.server.problems.UnprocessableEntityProblem
@@ -27,6 +28,11 @@ import io.airbyte.api.server.services.UserService
 import io.micronaut.http.annotation.Controller
 import java.time.OffsetDateTime
 import java.util.UUID
+import javax.ws.rs.DELETE
+import javax.ws.rs.GET
+import javax.ws.rs.HeaderParam
+import javax.ws.rs.Path
+import javax.ws.rs.PathParam
 import javax.ws.rs.core.Response
 
 @Controller(JOBS_PATH)
@@ -35,7 +41,13 @@ open class JobsController(
   private val userService: UserService,
   private val connectionService: ConnectionService,
 ) : JobsApi {
-  override fun cancelJob(jobId: Long, userInfo: String?): Response {
+
+  @DELETE
+  @Path("/{jobId}")
+  override fun cancelJob(
+    @PathParam("jobId") jobId: Long,
+    @HeaderParam(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?,
+  ): Response {
     val userId: UUID = userService.getUserIdFromUserInfoString(userInfo)
 
     val jobResponse: Any? = TrackingHelper.callWithTracker(
@@ -130,7 +142,12 @@ open class JobsController(
     }
   }
 
-  override fun getJob(jobId: Long, userInfo: String?): Response {
+  @GET
+  @Path("/{jobId}")
+  override fun getJob(
+    @PathParam("jobId") jobId: Long,
+    @HeaderParam(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?,
+  ): Response {
     val userId: UUID = userService.getUserIdFromUserInfoString(userInfo)
 
     val jobResponse: Any? = TrackingHelper.callWithTracker(

airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt

@@ -10,6 +10,7 @@ import io.airbyte.airbyte_api.model.generated.InitiateOauthRequest
 import io.airbyte.airbyte_api.model.generated.SourceCreateRequest
 import io.airbyte.airbyte_api.model.generated.SourcePatchRequest
 import io.airbyte.airbyte_api.model.generated.SourcePutRequest
+import io.airbyte.api.server.apiTracking.TrackingHelper
 import io.airbyte.api.server.constants.DELETE
 import io.airbyte.api.server.constants.GET
 import io.airbyte.api.server.constants.PATCH
@@ -18,7 +19,6 @@ import io.airbyte.api.server.constants.PUT
 import io.airbyte.api.server.constants.SOURCES_PATH
 import io.airbyte.api.server.constants.SOURCES_WITH_ID_PATH
 import io.airbyte.api.server.constants.SOURCE_TYPE
-import io.airbyte.api.server.helpers.TrackingHelper
 import io.airbyte.api.server.helpers.getIdFromName
 import io.airbyte.api.server.helpers.getLocalUserInfoIfNull
 import io.airbyte.api.server.helpers.removeSourceTypeNode

airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt

@@ -13,9 +13,9 @@ import io.airbyte.airbyte_api.model.generated.StreamProperties
 import io.airbyte.api.client.model.generated.AirbyteStreamAndConfiguration
 import io.airbyte.api.client.model.generated.DestinationSyncMode
 import io.airbyte.api.client.model.generated.SyncMode
+import io.airbyte.api.server.apiTracking.TrackingHelper
 import io.airbyte.api.server.constants.GET
 import io.airbyte.api.server.constants.STREAMS_PATH
-import io.airbyte.api.server.helpers.TrackingHelper
 import io.airbyte.api.server.helpers.getLocalUserInfoIfNull
 import io.airbyte.api.server.problems.UnexpectedProblem
 import io.airbyte.api.server.services.DestinationService