operate-first · sesheta · Oct 7, 2022 · Oct 6, 2022
diff --git a/openmetadata/base/openmetadata-dependencies/configmaps/files/pod_template.yaml b/openmetadata/base/openmetadata-dependencies/configmaps/files/pod_template.yaml
@@ -8,6 +8,7 @@ metadata:
 spec:
   restartPolicy: Never
   serviceAccountName: airflow
+  shareProcessNamespace: false
   nodeSelector:
     {}
   affinity:
@@ -24,7 +25,7 @@ spec:
     []
   containers:
     - name: base
-      image: quay.io/operate-first/om-airflow:0.12.0
+      image: quay.io/os-climate/ingestion:0.12.1.trino.317
       imagePullPolicy: IfNotPresent
       envFrom:
         - secretRef:

diff --git a/openmetadata/base/openmetadata-dependencies/configmaps/mysql-init-scripts.yaml b/openmetadata/base/openmetadata-dependencies/configmaps/mysql-init-scripts.yaml
@@ -6,7 +6,7 @@ metadata:
     app.kubernetes.io/name: mysql
 data:
   init_airflow_db_scripts.sql: |
-    CREATE DATABASE {{ .AIRFLOW_DB }};
+    CREATE DATABASE {{ .AIRFLOW_DB }} CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     CREATE USER '{{ .AIRFLOW_DB_USER }}'@'%' IDENTIFIED BY '{{ .AIRFLOW_DB_PASSWORD }}';
     GRANT ALL PRIVILEGES ON {{ .AIRFLOW_DB }}.* TO '{{ .AIRFLOW_DB_USER }}'@'%' WITH GRANT OPTION;
     commit;

diff --git a/...data/base/openmetadata-dependencies/configmaps/openmetadata-dependencies-config-envs.yaml b/...data/base/openmetadata-dependencies/configmaps/openmetadata-dependencies-config-envs.yaml
@@ -68,17 +68,13 @@ data:
   ## Airflow Configs (Kubernetes)
   ## ================
   AIRFLOW__KUBERNETES__NAMESPACE: "openmetadata"
-  AIRFLOW__KUBERNETES__WORKER_CONTAINER_REPOSITORY: "quay.io/operate-first/om-airflow"
-  AIRFLOW__KUBERNETES__WORKER_CONTAINER_TAG: "0.12.0"
+  AIRFLOW__KUBERNETES__WORKER_CONTAINER_REPOSITORY: "quay.io/os-climate/ingestion"
+  AIRFLOW__KUBERNETES__WORKER_CONTAINER_TAG: "0.12.1.trino.317"
   AIRFLOW__KUBERNETES__POD_TEMPLATE_FILE: "/opt/airflow/pod_templates/pod_template.yaml"
   ## ================
   ## User Configs
   ## ================
   "AIRFLOW__API__AUTH_BACKENDS": "airflow.api.auth.backend.basic_auth"
-  "AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME": "airflow_helm"
-  "AIRFLOW__LINEAGE__AUTH_PROVIDER_TYPE": "no-auth"
-  "AIRFLOW__LINEAGE__BACKEND": "airflow_provider_openmetadata.lineage.openmetadata.OpenMetadataLineageBackend"
-  "AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT": "http://openmetadata.openmetadata.svc.cluster.local:8585/api"
   "AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS": "/opt/airflow/dags"
   "AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_ACCESS_KEY": ""
   "AIRFLOW__OPENMETADATA_SECRETS_MANAGER__AWS_ACCESS_KEY_ID": ""

diff --git a/openmetadata/base/openmetadata-dependencies/deployments/airflow-db-migrations.yaml b/openmetadata/base/openmetadata-dependencies/deployments/airflow-db-migrations.yaml
@@ -38,7 +38,7 @@ spec:
       serviceAccountName: airflow
       initContainers:
         - name: check-db
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
@@ -68,7 +68,7 @@ spec:
               mountPath: /opt/airflow/logs
       containers:
         - name: db-migrations
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           resources: {}
           envFrom:

diff --git a/openmetadata/base/openmetadata-dependencies/deployments/airflow-scheduler.yaml b/openmetadata/base/openmetadata-dependencies/deployments/airflow-scheduler.yaml
@@ -41,7 +41,7 @@ spec:
       serviceAccountName: airflow
       initContainers:
         - name: check-db
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
@@ -70,7 +70,7 @@ spec:
             - name: logs-data
               mountPath: /opt/airflow/logs
         - name: wait-for-db-migrations
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
@@ -100,7 +100,7 @@ spec:
               mountPath: /opt/airflow/logs
       containers:
         - name: airflow-scheduler
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           resources: {}
           envFrom:

diff --git a/openmetadata/base/openmetadata-dependencies/deployments/airflow-sync-users.yaml b/openmetadata/base/openmetadata-dependencies/deployments/airflow-sync-users.yaml
@@ -38,7 +38,7 @@ spec:
       serviceAccountName: airflow
       initContainers:
         - name: check-db
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
@@ -67,7 +67,7 @@ spec:
             - name: logs-data
               mountPath: /opt/airflow/logs
         - name: wait-for-db-migrations
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
@@ -97,7 +97,7 @@ spec:
               mountPath: /opt/airflow/logs
       containers:
         - name: sync-airflow-users
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           resources: {}
           envFrom:

diff --git a/openmetadata/base/openmetadata-dependencies/deployments/airflow-web.yaml b/openmetadata/base/openmetadata-dependencies/deployments/airflow-web.yaml
@@ -32,7 +32,7 @@ spec:
       serviceAccountName: airflow
       initContainers:
         - name: check-db
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
@@ -63,7 +63,7 @@ spec:
             - name: logs-data
               mountPath: /opt/airflow/logs
         - name: wait-for-db-migrations
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
@@ -93,7 +93,7 @@ spec:
               mountPath: /opt/airflow/logs
       containers:
         - name: airflow-web
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           resources: {}
           ports:

diff --git a/openmetadata/base/openmetadata-dependencies/deployments/airlfow-triggerer.yaml b/openmetadata/base/openmetadata-dependencies/deployments/airlfow-triggerer.yaml
@@ -41,7 +41,7 @@ spec:
       serviceAccountName: airflow
       initContainers:
         - name: check-db
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
@@ -70,7 +70,7 @@ spec:
             - name: logs-data
               mountPath: /opt/airflow/logs
         - name: wait-for-db-migrations
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
@@ -100,7 +100,7 @@ spec:
               mountPath: /opt/airflow/logs
       containers:
         - name: airflow-triggerer
-          image: quay.io/operate-first/om-airflow:0.12.0
+          image: quay.io/operate-first/om-ingestion:0.12.1
           imagePullPolicy: IfNotPresent
           resources: {}
           envFrom:
@@ -127,8 +127,8 @@ spec:
           livenessProbe:
             initialDelaySeconds: 10
             periodSeconds: 30
-            failureThreshold: 5
             timeoutSeconds: 60
+            failureThreshold: 5
             exec:
               command:
                 - "/usr/bin/dumb-init"

diff --git a/openmetadata/base/openmetadata-dependencies/poddisruptionbudgets/elasticsearch-pdb.yaml b/openmetadata/base/openmetadata-dependencies/poddisruptionbudgets/elasticsearch-pdb.yaml
diff --git a/openmetadata/base/openmetadata-dependencies/poddisruptionbudgets/kustomization.yaml b/openmetadata/base/openmetadata-dependencies/poddisruptionbudgets/kustomization.yaml
diff --git a/openmetadata/base/openmetadata-dependencies/statefulsets/mysql.yaml b/openmetadata/base/openmetadata-dependencies/statefulsets/mysql.yaml
@@ -127,4 +127,4 @@ spec:
           - "ReadWriteOnce"
         resources:
           requests:
-            storage: "8Gi"
+            storage: "50Gi"
diff --git a/openmetadata/base/openmetadata/configmaps/files/openmetadata.yaml b/openmetadata/base/openmetadata/configmaps/files/openmetadata.yaml
@@ -12,7 +12,7 @@
 clusterName: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
 
 swagger:
-  resourcePackage: org.openmetadata.catalog.resources
+  resourcePackage: org.openmetadata.service.resources
 
 
 server:
@@ -91,9 +91,13 @@ server:
 logging:
   level: ${LOG_LEVEL:-DEBUG}
   loggers:
-    org.openmetadata.catalog.events: DEBUG
+    org.openmetadata.service.events: DEBUG
     io.swagger: ERROR
   appenders:
+    - type: console
+      threshold: TRACE
+      logFormat: "%level [%d{HH:mm:ss.SSS}] [%t] %logger{5} - %msg %n"
+      timeZone: UTC
     - type: file
       filterFactories:
         - type: audit-exclude-filter-factory
@@ -130,27 +134,29 @@ migrationConfiguration:
 
 # Authorizer Configuration
 authorizerConfiguration:
-  className: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.catalog.security.NoopAuthorizer}
-  containerRequestFilter: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.catalog.security.NoopFilter}
+  className: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
+  containerRequestFilter: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
   adminPrincipals: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
+  allowedEmailRegistrationDomains: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
   botPrincipals: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
   principalDomain: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"}
   enforcePrincipalDomain: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
   enableSecureSocketConnection: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
 
 authenticationConfiguration:
-  provider: ${AUTHENTICATION_PROVIDER:-no-auth}
+  provider: ${AUTHENTICATION_PROVIDER:-basic}
   # This will only be valid when provider type specified is customOidc
   providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
   publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[https://www.googleapis.com/oauth2/v3/certs]}
   authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
   clientId: ${AUTHENTICATION_CLIENT_ID:-""}
   callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
   jwtPrincipalClaims: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
+  enableSelfSignup: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
 
 jwtTokenConfiguration:
-  rsapublicKeyFilePath: ${RSA_PUBLIC_KEY_FILE_PATH:-""}
-  rsaprivateKeyFilePath: ${RSA_PRIVATE_KEY_FILE_PATH:-""}
+  rsapublicKeyFilePath: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
+  rsaprivateKeyFilePath: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
   jwtissuer: ${JWT_ISSUER:-"open-metadata.org"}
   keyId: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
 
@@ -168,8 +174,8 @@ elasticsearch:
 
 eventHandlerConfiguration:
   eventHandlerClassNames:
-    - "org.openmetadata.catalog.events.AuditEventHandler"
-    - "org.openmetadata.catalog.events.ChangeEventHandler"
+    - "org.openmetadata.service.events.AuditEventHandler"
+    - "org.openmetadata.service.events.ChangeEventHandler"
 
 airflowConfiguration:
   apiEndpoint: ${AIRFLOW_HOST:-http://localhost:8080}
@@ -203,6 +209,10 @@ airflowConfiguration:
       tokenEndpoint: ${OM_AUTH_AIRFLOW_CUSTOM_OIDC_TOKEN_ENDPOINT_URL:-""}
     openmetadata:
       jwtToken: ${OM_AUTH_JWT_TOKEN:-""}
+  verifySSL: ${AIRFLOW_VERIFY_SSL:-"no-ssl"}  # Possible values are "no-ssl", "ignore", "validate"
+  sslConfig:
+    validate:
+      certificatePath: ${AIRFLOW_SSL_CERT_PATH:-""}   # Local path for Airflow
 
 # no_encryption_at_rest is the default value, and it does what it says. Please read the manual on how
 # to secure your instance of OpenMetadata with TLS and encryption at rest.
@@ -229,6 +239,21 @@ health:
         failureAttempts: 2
         successAttempts: 1
 
+email:
+  emailingEntity: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
+  supportUrl: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
+  enableSmtpServer: ${AUTHORIZER_ENABLE_SMTP:-false}
+  openMetadataUrl: ${OPENMETADATA_SERVER_URL:-""}
+  serverEndpoint: ${SMTP_SERVER_ENDPOINT:-""}
+  serverPort: ${SMTP_SERVER_PORT:-""}
+  username: ${SMTP_SERVER_USERNAME:-""}
+  password: ${SMTP_SERVER_PWD:-""}
+  transportationStrategy: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
+
 sandboxModeEnabled: ${SANDBOX_MODE_ENABLED:-false}
 slackChat:
   slackUrl: ${SLACK_CHAT_SLACK_URL:-""}
+
+login:
+  maxLoginFailAttempts: ${OM_MAX_FAILED_LOGIN_ATTEMPTS:-3}
+  accessBlockTime: ${OM_LOGIN_ACCESS_BLOCKTIME:-600}
diff --git a/openmetadata/base/openmetadata/deployments/openmetadata.yaml b/openmetadata/base/openmetadata/deployments/openmetadata.yaml
@@ -15,9 +15,13 @@ spec:
         app.kubernetes.io/name: openmetadata
     spec:
       serviceAccountName: openmetadata
+      securityContext: {}
+      initContainers: []
+      volumes: []
       containers:
         - name: openmetadata
-          image: "quay.io/operate-first/om-server:0.12.0"
+          securityContext: {}
+          image: "quay.io/operate-first/om-server:0.12.1"
           imagePullPolicy: Always
           ports:
             - name: http
@@ -52,21 +56,27 @@ spec:
             - name: SERVER_ADMIN_PORT
               value: "8586"
             - name: AUTHENTICATION_PROVIDER
-              value: "no-auth"
+              value: "basic"
             - name: AUTHENTICATION_PUBLIC_KEYS
-              value: "[]"
+              value: "[http://openmetadata:8585/api/v1/config/jwks]"
             - name: AUTHENTICATION_AUTHORITY
-              value: ""
+              value: "https://accounts.google.com"
             - name: AUTHENTICATION_CLIENT_ID
               value: ""
             - name: AUTHENTICATION_CALLBACK_URL
               value: ""
             - name: AUTHENTICATION_JWT_PRINCIPAL_CLAIMS
               value: "[email,preferred_username,sub]"
+            - name: AUTHENTICATION_ENABLE_SELF_SIGNUP
+              value: "true"
+            - name: OM_MAX_FAILED_LOGIN_ATTEMPTS
+              value: "3"
+            - name: OM_LOGIN_ACCESS_BLOCKTIME
+              value: "600"
             - name: AUTHORIZER_CLASS_NAME
-              value: "org.openmetadata.catalog.security.NoopAuthorizer"
+              value: "org.openmetadata.service.security.DefaultAuthorizer"
             - name: AUTHORIZER_REQUEST_FILTER
-              value: "org.openmetadata.catalog.security.NoopFilter"
+              value: "org.openmetadata.service.security.JwtFilter"
             - name: AUTHORIZER_ADMIN_PRINCIPALS
               value: "[admin]"
             - name: AUTHORIZER_INGESTION_PRINCIPALS
@@ -77,6 +87,19 @@ spec:
               value: "false"
             - name: AUTHORIZER_ENABLE_SECURE_SOCKET
               value: "false"
+            - name: AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN
+              value: "[all]"
+            - name: RSA_PUBLIC_KEY_FILE_PATH
+              value: "./conf/public_key.der"
+            - name: RSA_PRIVATE_KEY_FILE_PATH
+              value: "./conf/private_key.der"
+            - name: JWT_ISSUER
+              value: "open-metadata.org"
+            # We add these via a secret
+            # - name: JWT_KEY_ID
+            #  value: ""
+            # - name: FERNET_KEY
+            #  value: ""
             - name: ELASTICSEARCH_HOST
               value: "elasticsearch"
             - name: ELASTICSEARCH_PORT
@@ -110,6 +133,10 @@ spec:
                 secretKeyRef:
                   name: airflow-secrets
                   key: openmetadata-airflow-password
+            - name: AIRFLOW_VERIFY_SSL
+              value: "no-ssl"
+            - name: AIRFLOW_SSL_CERT_PATH
+              value: "/no/path"
             - name: SERVER_HOST_API_URL
               value: "http://openmetadata.default.svc.cluster.local:8585/api"
             - name: AIRFLOW_AUTH_PROVIDER

diff --git a/openmetadata/overlays/osc/osc-cl2/kustomization.yaml b/openmetadata/overlays/osc/osc-cl2/kustomization.yaml
@@ -16,3 +16,8 @@ patchesStrategicMerge:
   - patches/deployments/airflow-sync-users.yaml
   - patches/deployments/airflow-scheduler.yaml
   - patches/deployments/airflow-db-migrations.yaml
+
+images:
+  - name: quay.io/operate-first/om-ingestion
+    newName: quay.io/os-climate/ingestion
+    newTag: 0.12.1.trino.317