Merge pull request #2961 from elifesciences/add-config-for-materializ…

…e-bq-views-for-kubernetes

Add config for 'Materialize_BigQuery_Views_Pipeline_Kubernetes' (prod)

kustomizations/apps/data-hub-configs/kubernetes-pipeline--prod.config.yaml

@@ -265,3 +265,57 @@ kubernetesPipelines:
       requests:
         memory: 1Gi
         cpu: 100m
+
+
+  - dataPipelineId: 'Materialize_BigQuery_Views_Pipeline_Kubernetes'
+    airflow:
+      dagParameters:
+        schedule: '@hourly'
+        tags:
+          - 'Kubernetes'
+    image: 'docker.io/elifesciences/data-hub-core-dags:latest'
+    imagePullPolicy: Always
+    arguments: 
+      - 'python'
+      - '-m'
+      - 'data_pipeline.bigquery_views.cli'
+    env:
+      - name: DEPLOYMENT_ENV
+        value: '{ENV}'
+      - name: GOOGLE_APPLICATION_CREDENTIALS
+        value: /dag_secret_files/gcloud/credentials.json
+      - name: AWS_CONFIG_FILE
+        value: /dag_secret_files/aws/credentials
+      - name: MATERIALIZE_BIGQUERY_VIEWS_CONFIG_PATH
+        value: "s3://staging-elife-data-pipeline/airflow-config/bigquery-views"
+      - name: MATERIALIZE_BIGQUERY_VIEWS_GCP_PROJECT
+        value: "elife-data-pipeline"
+      - name: MATERIALIZE_BIGQUERY_VIEWS_DATASET
+        value: prod
+    volumeMounts:
+      - name: gcloud-secret-volume
+        mountPath: /dag_secret_files/gcloud/
+        readOnly: true
+      - name: aws-secret-volume
+        mountPath: /dag_secret_files/aws
+        readOnly: true
+      - name: data-hub-config-volume
+        mountPath: /dag_config_files/
+        readOnly: true
+    volumes:
+      - name: aws-secret-volume
+        secret:
+          secretName: credentials
+      - name: gcloud-secret-volume
+        secret:
+          secretName: gcloud
+      - name: data-hub-config-volume
+        configMap:
+          name: data-hub-configs
+    resources:
+      limits:
+        memory: 1Gi
+        cpu: 1000m
+      requests:
+        memory: 1Gi
+        cpu: 100m