Merge pull request #52 from IATI/develop

Datastore Services to Production
IATI · Nov 12, 2021 · ff87564 · ff87564
2 parents f736e2f + 04c22e2
commit ff87564
Show file tree

Hide file tree

Showing 15 changed files with 585 additions and 88 deletions.
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -14,14 +14,21 @@ env:
   FILE_VALIDATION_URL: ${{ secrets.FILE_VALIDATION_URL }}
   FILE_VALIDATION_KEY_NAME: ${{ secrets.DEV_FILE_VALIDATION_KEY_NAME }}
   FILE_VALIDATION_KEY_VALUE: ${{ secrets.DEV_FILE_VALIDATION_KEY_VALUE }}
-  FLATTENER_API_URL: ${{ secrets.FLATTENER_API_URL }}
+  FLATTENER_API_URL: ${{ secrets.DEV_FLATTENER_API_URL }}
+  FLATTENER_KEY_NAME: ${{ secrets.DEV_FLATTENER_KEY_NAME }}
+  FLATTENER_KEY_VALUE: ${{ secrets.DEV_FLATTENER_API_KEY }}
   SOLR_API_URL: ${{ secrets.SOLR_API_URL }}
+  SOLR_USER: ${{ secrets.SOLR_USER }}
+  SOLR_PASSWORD: ${{ secrets.SOLR_PASSWORD }}
   VALIDATOR_PARALLEL: ${{ secrets.VALIDATOR_PARALLEL }}
   DB_USER: ${{ secrets.DB_USER }}
   DB_PASS: ${{ secrets.DB_PASS }}
   DB_HOST: ${{ secrets.DB_HOST }}
   DB_NAME: ${{ secrets.DB_NAME }}
   DB_PORT: ${{ secrets.DB_PORT }}
+  LOG_WORKSPACE_ID: ${{ secrets.DEV_LOG_WORKSPACE_ID }}
+  LOG_WORKSPACE_KEY: ${{ secrets.DEV_LOG_WORKSPACE_KEY }}
+  ACTIVITIES_LAKE_CONTAINER_NAME: ${{ secrets.DEV_ACTIVITIES_LAKE_CONTAINER_NAME }}
 
 jobs:
   build-and-deploy:
@@ -48,10 +55,12 @@ jobs:
           docker push ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-flatten:$TAG
           docker build . -f solrize-dockerfile  -t ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-solrize:$TAG
           docker push ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-solrize:$TAG
+          docker build . -f lakify-dockerfile  -t ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-lakify:$TAG
+          docker push ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-lakify:$TAG
       - name: "Deploy Refresh to Azure Container Instances"
-        uses: "azure/CLI@v1"
+        uses: "azure/CLI@1.0.4"
         with:
-            azcliversion: 2.19.1
+            azcliversion: 2.30.0
             inlineScript: |
               az container create \
               --resource-group "${{ secrets.RESOURCE_GROUP }}" \
@@ -68,6 +77,8 @@ jobs:
               --memory "4" \
               --cpu "1" \
               --os-type "Linux" \
+              --log-analytics-workspace "${{ env.LOG_WORKSPACE_ID }}" \
+              --log-analytics-workspace-key "${{ env.LOG_WORKSPACE_KEY }}" \
               --secure-environment-variables \
                 AZURE_STORAGE_CONNECTION_STRING="${{ env.AZURE_STORAGE_CONNECTION_STRING }}" \
                 AZURE_STORAGE_CONTAINER_SOURCE="${{ env.AZURE_STORAGE_CONTAINER_SOURCE }}" \
@@ -76,11 +87,15 @@ jobs:
                 DB_USER="${{ env.DB_USER }}" \
                 DB_PASS="${{ env.DB_PASS }}" \
                 DB_NAME="${{ env.DB_NAME }}" \
-                SOLR_API_URL="${{ env.SOLR_API_URL }}" 
+                SOLR_API_URL="${{ env.SOLR_API_URL }}" \
+                SOLR_USER="${{ env.SOLR_USER }}" \
+                SOLR_PASSWORD="${{ env.SOLR_PASSWORD }}" \
+                ACTIVITIES_LAKE_CONTAINER_NAME="${{ env.ACTIVITIES_LAKE_CONTAINER_NAME }}"
+                
       - name: "Deploy Validate to Azure Container Instances"
-        uses: "azure/CLI@v1"
+        uses: "azure/CLI@1.0.4"
         with:
-            azcliversion: 2.19.1
+            azcliversion: 2.30.0
             inlineScript: |
               az container create \
               --resource-group "${{ secrets.RESOURCE_GROUP }}" \
@@ -97,6 +112,8 @@ jobs:
               --memory "2" \
               --cpu "1" \
               --os-type "Linux" \
+              --log-analytics-workspace "${{ env.LOG_WORKSPACE_ID }}" \
+              --log-analytics-workspace-key "${{ env.LOG_WORKSPACE_KEY }}" \
               --secure-environment-variables \
                 AZURE_STORAGE_CONNECTION_STRING="${{ env.AZURE_STORAGE_CONNECTION_STRING }}" \
                 AZURE_STORAGE_CONTAINER_SOURCE="${{ env.AZURE_STORAGE_CONTAINER_SOURCE }}" \
@@ -109,11 +126,14 @@ jobs:
                 DB_USER="${{ env.DB_USER }}" \
                 DB_PASS="${{ env.DB_PASS }}" \
                 DB_NAME="${{ env.DB_NAME }}" \
-                SOLR_API_URL="${{ env.SOLR_API_URL }}"
+                SOLR_API_URL="${{ env.SOLR_API_URL }}" \
+                SOLR_USER="${{ env.SOLR_USER }}" \
+                SOLR_PASSWORD="${{ env.SOLR_PASSWORD }}"
+     
       - name: "Deploy Flatten to Azure Container Instances"
-        uses: "azure/CLI@v1"
+        uses: "azure/CLI@1.0.4"
         with:
-            azcliversion: 2.19.1
+            azcliversion: 2.30.0
             inlineScript: |
               az container create \
               --resource-group "${{ secrets.RESOURCE_GROUP }}" \
@@ -130,20 +150,27 @@ jobs:
               --memory "2" \
               --cpu "1" \
               --os-type "Linux" \
+              --log-analytics-workspace "${{ env.LOG_WORKSPACE_ID }}" \
+              --log-analytics-workspace-key "${{ env.LOG_WORKSPACE_KEY }}" \
               --secure-environment-variables \
                 AZURE_STORAGE_CONNECTION_STRING="${{ env.AZURE_STORAGE_CONNECTION_STRING }}" \
                 AZURE_STORAGE_CONTAINER_SOURCE="${{ env.AZURE_STORAGE_CONTAINER_SOURCE }}" \
                 FLATTENER_API_URL="${{ env.FLATTENER_API_URL }}" \
+                FLATTENER_KEY_NAME="${{ env.FLATTENER_KEY_NAME }}" \
+                FLATTENER_KEY_VALUE="${{ env.FLATTENER_KEY_VALUE }}" \
                 DB_HOST="${{ env.DB_HOST }}" \
                 DB_PORT="${{ env.DB_PORT }}" \
                 DB_USER="${{ env.DB_USER }}" \
                 DB_PASS="${{ env.DB_PASS }}" \
                 DB_NAME="${{ env.DB_NAME }}" \
-                SOLR_API_URL="${{ env.SOLR_API_URL }}"
+                SOLR_API_URL="${{ env.SOLR_API_URL }}" \
+                SOLR_USER="${{ env.SOLR_USER }}" \
+                SOLR_PASSWORD="${{ env.SOLR_PASSWORD }}"
+                
       - name: "Deploy Solrize to Azure Container Instances"
-        uses: "azure/CLI@v1"
+        uses: "azure/CLI@1.0.4"
         with:
-            azcliversion: 2.19.1
+            azcliversion: 2.30.0
             inlineScript: |
               az container create \
               --resource-group "${{ secrets.RESOURCE_GROUP }}" \
@@ -157,12 +184,50 @@ jobs:
               --ip-address "Private" \
               --vnet "dev-iati-refresher" \
               --subnet "default" \
+              --memory "4" \
+              --cpu "1" \
+              --os-type "Linux" \
+              --log-analytics-workspace "${{ env.LOG_WORKSPACE_ID }}" \
+              --log-analytics-workspace-key "${{ env.LOG_WORKSPACE_KEY }}" \
+              --secure-environment-variables \
+                AZURE_STORAGE_CONNECTION_STRING="${{ env.AZURE_STORAGE_CONNECTION_STRING }}" \
+                AZURE_STORAGE_CONTAINER_SOURCE="${{ env.AZURE_STORAGE_CONTAINER_SOURCE }}" \
+                SOLR_API_URL="${{ env.SOLR_API_URL }}" \
+                SOLR_USER="${{ env.SOLR_USER }}" \
+                SOLR_PASSWORD="${{ env.SOLR_PASSWORD }}" \
+                ACTIVITIES_LAKE_CONTAINER_NAME="${{ env.ACTIVITIES_LAKE_CONTAINER_NAME }}" \
+                DB_HOST="${{ env.DB_HOST }}" \
+                DB_PORT="${{ env.DB_PORT }}" \
+                DB_USER="${{ env.DB_USER }}" \
+                DB_PASS="${{ env.DB_PASS }}" \
+                DB_NAME="${{ env.DB_NAME }}"
+
+      - name: "Deploy Lakify to Azure Container Instances"
+        uses: "azure/[email protected]"
+        with:
+            azcliversion: 2.30.0
+            inlineScript: |
+              az container create \
+              --resource-group "${{ secrets.RESOURCE_GROUP }}" \
+              --image "${{ secrets.REGISTRY_LOGIN_SERVER }}/${{env.STAGE}}-${{env.NAME}}-lakify:${{ env.TAG }}" \
+              --registry-login-server "${{ secrets.REGISTRY_LOGIN_SERVER }}" \
+              --registry-username "${{ secrets.REGISTRY_USERNAME }}" \
+              --registry-password "${{ secrets.REGISTRY_PASSWORD }}" \
+              --name "${{ env.STAGE }}-${{ env.NAME }}-lakify" \
+              --location "uksouth" \
+              --restart-policy "Always" \
+              --ip-address "Private" \
+              --vnet "dev-iati-refresher" \
+              --subnet "default" \
               --memory "2" \
               --cpu "1" \
               --os-type "Linux" \
+              --log-analytics-workspace "${{ env.LOG_WORKSPACE_ID }}" \
+              --log-analytics-workspace-key "${{ env.LOG_WORKSPACE_KEY }}" \
               --secure-environment-variables \
                 AZURE_STORAGE_CONNECTION_STRING="${{ env.AZURE_STORAGE_CONNECTION_STRING }}" \
                 AZURE_STORAGE_CONTAINER_SOURCE="${{ env.AZURE_STORAGE_CONTAINER_SOURCE }}" \
+                ACTIVITIES_LAKE_CONTAINER_NAME="${{ env.ACTIVITIES_LAKE_CONTAINER_NAME }}" \
                 SOLR_API_URL="${{ env.SOLR_API_URL }}" \
                 DB_HOST="${{ env.DB_HOST }}" \
                 DB_PORT="${{ env.DB_PORT }}" \

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -13,8 +13,12 @@ env:
   FILE_VALIDATION_URL: ${{ secrets.PROD_FILE_VALIDATION_URL }}
   FILE_VALIDATION_KEY_NAME: ${{ secrets.PROD_FILE_VALIDATION_KEY_NAME }}
   FILE_VALIDATION_KEY_VALUE: ${{ secrets.PROD_FILE_VALIDATION_KEY_VALUE }}
-  FLATTENER_API_URL: ${{ secrets.FLATTENER_API_URL }}
-  SOLR_API_URL: ${{ secrets.SOLR_API_URL }}
+  FLATTENER_API_URL: ${{ secrets.PROD_FLATTENER_API_URL }}
+  FLATTENER_KEY_NAME: ${{ secrets.PROD_FLATTENER_KEY_NAME }}
+  FLATTENER_KEY_VALUE: ${{ secrets.PROD_FLATTENER_API_KEY }}
+  SOLR_API_URL: ${{ secrets.PROD_SOLR_API_URL }}
+  SOLR_USER: ${{ secrets.PROD_SOLR_USER }}
+  SOLR_PASSWORD: ${{ secrets.PROD_SOLR_PASSWORD }}
   VALIDATOR_PARALLEL: ${{ secrets.VALIDATOR_PARALLEL }}
   DB_USER: ${{ secrets.PROD_DB_USER }}
   DB_PASS: ${{ secrets.PROD_DB_PASS }}
@@ -45,6 +49,8 @@ jobs:
           docker push ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-refresh:$TAG
           docker build . -f validate-dockerfile  -t ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-validate:$TAG
           docker push ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-validate:$TAG
+          docker build . -f flatten-dockerfile  -t ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-flatten:$TAG
+          docker push ${{ secrets.REGISTRY_LOGIN_SERVER }}/$STAGE-$NAME-flatten:$TAG
       - name: "Deploy Refresh to Azure Container Instances"
         uses: "azure/CLI@v1"
         with:
@@ -71,6 +77,8 @@ jobs:
                 AZURE_STORAGE_CONNECTION_STRING="${{ env.AZURE_STORAGE_CONNECTION_STRING }}" \
                 AZURE_STORAGE_CONTAINER_SOURCE="${{ env.AZURE_STORAGE_CONTAINER_SOURCE }}" \
                 SOLR_API_URL="${{ env.SOLR_API_URL }}" \
+                SOLR_USER="${{ env.SOLR_USER }}" \
+                SOLR_PASSWORD="${{ env.SOLR_PASSWORD }}" \
                 DB_HOST="${{ env.DB_HOST }}" \
                 DB_PORT="${{ env.DB_PORT }}" \
                 DB_USER="${{ env.DB_USER }}" \
@@ -111,4 +119,44 @@ jobs:
                 DB_USER="${{ env.DB_USER }}" \
                 DB_PASS="${{ env.DB_PASS }}" \
                 DB_NAME="${{ env.DB_NAME }}" \
-                SOLR_API_URL="${{ env.SOLR_API_URL }}"
+                SOLR_API_URL="${{ env.SOLR_API_URL }}" \
+                SOLR_USER="${{ env.SOLR_USER }}" \
+                SOLR_PASSWORD="${{ env.SOLR_PASSWORD }}"
+
+      - name: "Deploy Flatten to Azure Container Instances"
+        uses: "azure/[email protected]"
+        with:
+            azcliversion: 2.30.0
+            inlineScript: |
+              az container create \
+              --resource-group "${{ secrets.RESOURCE_GROUP }}" \
+              --image "${{ secrets.REGISTRY_LOGIN_SERVER }}/${{env.STAGE}}-${{env.NAME}}-flatten:${{ env.TAG }}" \
+              --registry-login-server "${{ secrets.REGISTRY_LOGIN_SERVER }}" \
+              --registry-username "${{ secrets.REGISTRY_USERNAME }}" \
+              --registry-password "${{ secrets.REGISTRY_PASSWORD }}" \
+              --name "${{ env.STAGE }}-${{ env.NAME }}-flatten" \
+              --location "uksouth" \
+              --restart-policy "Always" \
+              --ip-address "Private" \
+              --vnet "dev-iati-refresher" \
+              --subnet "default" \
+              --memory "2" \
+              --cpu "1" \
+              --os-type "Linux" \
+              --log-analytics-workspace "${{ env.LOG_WORKSPACE_ID }}" \
+              --log-analytics-workspace-key "${{ env.LOG_WORKSPACE_KEY }}" \
+              --secure-environment-variables \
+                AZURE_STORAGE_CONNECTION_STRING="${{ env.AZURE_STORAGE_CONNECTION_STRING }}" \
+                AZURE_STORAGE_CONTAINER_SOURCE="${{ env.AZURE_STORAGE_CONTAINER_SOURCE }}" \
+                FLATTENER_API_URL="${{ env.FLATTENER_API_URL }}" \
+                FLATTENER_KEY_NAME="${{ env.FLATTENER_KEY_NAME }}" \
+                FLATTENER_KEY_VALUE="${{ env.FLATTENER_KEY_VALUE }}" \
+                DB_HOST="${{ env.DB_HOST }}" \
+                DB_PORT="${{ env.DB_PORT }}" \
+                DB_USER="${{ env.DB_USER }}" \
+                DB_PASS="${{ env.DB_PASS }}" \
+                DB_NAME="${{ env.DB_NAME }}" \
+                SOLR_API_URL="${{ env.SOLR_API_URL }}" \
+                SOLR_USER="${{ env.SOLR_USER }}" \
+                SOLR_PASSWORD="${{ env.SOLR_PASSWORD }}"
+              
diff --git a/lakify-dockerfile b/lakify-dockerfile
@@ -0,0 +1,7 @@
+FROM python:3.8
+RUN apt-get update && apt-get -y install cron
+WORKDIR /code
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+COPY src/ .
+CMD /usr/local/bin/python /code/handler.py -t lakifyloop
diff --git a/src/constants/config.py b/src/constants/config.py
@@ -3,6 +3,7 @@
 config = dict(DATA_SCHEMA = "public",
     PARALLEL_PROCESSES = 10,
     SOURCE_CONTAINER_NAME = os.getenv('AZURE_STORAGE_CONTAINER_SOURCE'),
+    ACTIVITIES_LAKE_CONTAINER_NAME = os.getenv('ACTIVITIES_LAKE_CONTAINER_NAME'),
     STORAGE_CONNECTION_STR = os.getenv('AZURE_STORAGE_CONNECTION_STRING'),
     SERVICE_LOOP_SLEEP = 60,
     RETRY_ERRORS_AFTER_LOOP = 600,
@@ -20,10 +21,18 @@
     ),
     FLATTEN = dict(
         PARALLEL_PROCESSES = 1,
-        FLATTENER_URL = os.getenv('FLATTENER_API_URL')
+        FLATTENER_URL = os.getenv('FLATTENER_API_URL'),
+        FLATTENER_KEY_NAME = os.getenv('FLATTENER_KEY_NAME'),
+        FLATTENER_KEY_VALUE = os.getenv('FLATTENER_KEY_VALUE')
     ),
     SOLRIZE = dict(
-        PARALLEL_PROCESSES = 10,
-        SOLR_API_URL = os.getenv('SOLR_API_URL')
+        PARALLEL_PROCESSES = 20,
+        SOLR_API_URL = os.getenv('SOLR_API_URL'),
+        SOLR_USER = os.getenv('SOLR_USER'),
+        SOLR_PASSWORD = os.getenv('SOLR_PASSWORD'),
+        EXPLODE_ELEMENTS = '["transaction", "budget"]'
+    ),
+    LAKIFY = dict(
+        PARALLEL_PROCESSES = 10
     )
 )
diff --git a/src/constants/version.py b/src/constants/version.py
@@ -1 +1 @@
-__version__ = {'number': "0.1.12", 'migration': 12}
+__version__ = {'number': "0.1.14", 'migration': 14}
diff --git a/src/handler.py b/src/handler.py
@@ -3,6 +3,7 @@
 import library.validate as validate
 import library.validate_adhoc as validate_adhoc
 import library.flatten as flatten
+import library.lakify as lakify
 import library.solrize as solrize
 import library.db as db
 from datetime import datetime, timedelta
@@ -28,6 +29,8 @@ def main(args):
             validate_adhoc.main()
         elif args.type == "flatten":
             flatten.main()
+        elif args.type == "lakify":
+            lakify.main()
         elif args.type == "solrize":
             solrize.main()
         elif args.type == "validateloop":
@@ -36,10 +39,12 @@ def main(args):
             validate_adhoc.service_loop()
         elif args.type == "flattenloop":
             flatten.service_loop()
+        elif args.type == "lakifyloop":
+            lakify.service_loop()           
         elif args.type == "solrizeloop":
             solrize.service_loop()
         else:
-            print("Type is required - either refresh, reload, validate - or refreshLoop or validateLoop.")
+            print("Type is required - either refresh, reload, validate, flatten, lakify, or solrize - or their related service loop.")
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Refresh from IATI Registry')
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		__version__ = {'number': "0.1.12", 'migration': 12}
		__version__ = {'number': "0.1.14", 'migration': 14}