chore: remove git pulls at dags and update docker images at DockerOpe…

…rator calls (et/somenergia-jardiner!8) * remove legacy git pull, change to harbor, change image * remove git code updates from alarms
Som-Energia · Sep 5, 2023 · ff1c7de · ff1c7de
1 parent 3828f7c
commit ff1c7de
Show file tree

Hide file tree

Showing 9 changed files with 31 additions and 167 deletions.
diff --git a/Makefile b/Makefile
@@ -19,6 +19,9 @@ sh: ## run a shell in the container
 app.build: ## build image using docker build
 	@docker compose -f $(app_compose_file) --env-file $(app_compose_env_file) build app --progress=plain
 
+app.push: ## push image using docker push
+	@docker compose -f $(app_compose_file) --env-file $(app_compose_env_file) push app 
+
 app_dev.build: ## build image for development using docker build
 	@docker compose -f $(app_compose_file) --env-file $(app_compose_env_file) build app-dev --progress=plain
 

diff --git a/containers/app/Dockerfile b/containers/app/Dockerfile
@@ -120,7 +120,7 @@ FROM python:${PYTHON_VERSION}-slim AS app
 # refresh global arguments
 ARG WORKDIR
 ARG USERNAME
-ARG USER_UID=1010
+ARG USER_UID
 ARG USER_GID
 ARG DBT_PROJECT_DIR_NAME
 
@@ -139,12 +139,17 @@ ENV PATH=/opt/pipx/bin:${WORKDIR}/.venv/bin:$PATH \
 RUN groupadd --gid $USER_GID "${USERNAME}" \
 	&& useradd --uid $USER_UID --gid $USER_GID -m "${USERNAME}"
 
+RUN apt-get update \
+	&& apt-get install -y vim
+
 # ------------------------------- app specific ------------------------------- #
 
 WORKDIR ${WORKDIR}
 
+#
 RUN chown -R ${USER_UID}:${USER_GID} ${WORKDIR}
 
+# copy /app/.venv from app-pre
 COPY --from=app-pre --chown=${USER_UID}:${USER_GID} ${WORKDIR} ${WORKDIR}
 
 USER ${USERNAME}
@@ -202,7 +207,6 @@ COPY --chown=${USER_UID}:${USER_GID} ./${DBT_PROJECT_DIR_NAME}/dbt_project.yml .
 RUN cd ${WORKDIR}/${DBT_PROJECT_DIR_NAME}/ && dbt deps --profiles-dir /home/${USERNAME}/.dbt
 
 
-
 USER ${USERNAME}
 
 # ---------------------------------------------------------------------------- #
@@ -242,7 +246,7 @@ RUN groupadd --gid ${USER_GID} "${USERNAME}" \
 # --------------------------- install dependencies --------------------------- #
 
 RUN apt-get update \
-	&& apt-get install -y git sudo iputils-ping wget
+	&& apt-get install -y git vim iputils-ping wget
 
 # ------------------------------- app specific ------------------------------- #
 

diff --git a/dags/jardineria_dag.py b/dags/jardineria_dag.py
@@ -5,10 +5,6 @@
 from airflow.models import Variable
 from airflow.providers.docker.operators.docker import DockerOperator
 from docker.types import DriverConfig, Mount
-from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task
-from util_tasks.t_check_repo import build_check_repo_task
-from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task
-from util_tasks.t_update_docker_image import build_update_image_task
 
 my_email = Variable.get("fail_email")
 addr = Variable.get("repo_server_url")
@@ -59,19 +55,12 @@ def get_random_moll():
 
     sampled_moll = get_random_moll()
 
-    task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name)
-    task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name)
-    task_branch_pull_ssh = build_branch_pull_ssh_task(
-        dag=dag, task_name="jardineria", repo_name=repo_name
-    )
-    task_update_image = build_update_image_task(dag=dag, repo_name=repo_name)
-
     notify_alarms_task = DockerOperator(
         api_version="auto",
         task_id="jardineria",
-        docker_conn_id="somenergia_registry",
-        image="{}/{}-requirements:latest".format(
-            "{{ conn.somenergia_registry.host }}", repo_name
+        docker_conn_id="somenergia_harbor_dades_registry",
+        image="{}/{}-app:latest".format(
+            "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name
         ),
         working_dir=f"/repos/{repo_name}",
         command=(
@@ -89,9 +78,3 @@ def get_random_moll():
         trigger_rule="none_failed",
     )
 
-    task_check_repo >> task_git_clone
-    task_check_repo >> task_branch_pull_ssh
-    task_git_clone >> task_update_image
-    task_branch_pull_ssh >> notify_alarms_task
-    task_branch_pull_ssh >> task_update_image
-    task_update_image >> notify_alarms_task
diff --git a/dags/plant_production_datasets_dag.py b/dags/plant_production_datasets_dag.py
@@ -6,10 +6,6 @@
 from airflow.models import Variable
 from airflow.providers.docker.operators.docker import DockerOperator
 from docker.types import DriverConfig, Mount
-from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task
-from util_tasks.t_check_repo import build_check_repo_task
-from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task
-from util_tasks.t_update_docker_image import build_update_image_task
 
 my_email = Variable.get("fail_email")
 addr = Variable.get("repo_server_url")
@@ -68,13 +64,6 @@ def dbapi_to_dict(dbapi: str):
 
     sampled_moll = get_random_moll()
 
-    task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name)
-    task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name)
-    task_branch_pull_ssh = build_branch_pull_ssh_task(
-        dag=dag, task_name="dbt_deps_task", repo_name=repo_name
-    )
-    task_update_image = build_update_image_task(dag=dag, repo_name=repo_name)
-
     dbapi = Variable.get("plantmonitor_db")
 
     dbapi_dict = dbapi_to_dict(dbapi)
@@ -87,31 +76,13 @@ def dbapi_to_dict(dbapi: str):
         "DBNAME": dbapi_dict["database"],
     }
 
-    dbt_deps_task = DockerOperator(
-        api_version="auto",
-        task_id="dbt_deps_task",
-        docker_conn_id="somenergia_registry",
-        environment=environment,
-        image="{}/{}-requirements:latest".format(
-            "{{ conn.somenergia_registry.host }}", repo_name
-        ),
-        working_dir=f"/repos/{repo_name}/dbt_jardiner",
-        command="dbt deps --profiles-dir config",
-        docker_url=sampled_moll,
-        mounts=[mount_nfs],
-        mount_tmp_dir=False,
-        auto_remove=True,
-        retrieve_output=True,
-        trigger_rule="none_failed",
-    )
-
     dbt_transformation_task = DockerOperator(
         api_version="auto",
         task_id="dbt_transformation_task",
-        docker_conn_id="somenergia_registry",
         environment=environment,
-        image="{}/{}-requirements:latest".format(
-            "{{ conn.somenergia_registry.host }}", repo_name
+        docker_conn_id="somenergia_harbor_dades_registry",
+        image="{}/{}-app:latest".format(
+            "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name
         ),
         working_dir=f"/repos/{repo_name}/dbt_jardiner",
         command="dbt run --profiles-dir config --target prod --select plant_production_hourly+",
@@ -122,11 +93,3 @@ def dbapi_to_dict(dbapi: str):
         retrieve_output=True,
         trigger_rule="none_failed",
     )
-
-    task_check_repo >> task_git_clone
-    task_check_repo >> task_branch_pull_ssh
-    task_git_clone >> task_update_image
-    task_branch_pull_ssh >> dbt_deps_task
-    task_branch_pull_ssh >> task_update_image
-    task_update_image >> dbt_deps_task
-    dbt_deps_task >> dbt_transformation_task
diff --git a/dags/plantmonitor_alert_inverter_interinverter_relative_temperature.py b/dags/plantmonitor_alert_inverter_interinverter_relative_temperature.py
@@ -5,10 +5,6 @@
 from airflow.models import Variable
 from airflow.providers.docker.operators.docker import DockerOperator
 from docker.types import DriverConfig, Mount
-from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task
-from util_tasks.t_check_repo import build_check_repo_task
-from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task
-from util_tasks.t_update_docker_image import build_update_image_task
 
 my_email = Variable.get("fail_email")
 addr = Variable.get("repo_server_url")
@@ -59,21 +55,12 @@ def get_random_moll():
 
     sampled_moll = get_random_moll()
 
-    task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name)
-    task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name)
-    task_branch_pull_ssh = build_branch_pull_ssh_task(
-        dag=dag,
-        task_name="alert_inverter_interinverter_relative_temperature",
-        repo_name=repo_name,
-    )
-    task_update_image = build_update_image_task(dag=dag, repo_name=repo_name)
-
     notify_alarms_task = DockerOperator(
         api_version="auto",
         task_id="alert_inverter_interinverter_relative_temperature",
-        docker_conn_id="somenergia_registry",
-        image="{}/{}-requirements:latest".format(
-            "{{ conn.somenergia_registry.host }}", repo_name
+        docker_conn_id="somenergia_harbor_dades_registry",
+        image="{}/{}-app:latest".format(
+            "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name
         ),
         working_dir=f"/repos/{repo_name}",
         command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\
@@ -86,9 +73,3 @@ def get_random_moll():
         trigger_rule="none_failed",
     )
 
-    task_check_repo >> task_git_clone
-    task_check_repo >> task_branch_pull_ssh
-    task_git_clone >> task_update_image
-    task_branch_pull_ssh >> notify_alarms_task
-    task_branch_pull_ssh >> task_update_image
-    task_update_image >> notify_alarms_task
diff --git a/dags/plantmonitor_alert_inverter_temperature.py b/dags/plantmonitor_alert_inverter_temperature.py
@@ -5,10 +5,6 @@
 from airflow.models import Variable
 from airflow.providers.docker.operators.docker import DockerOperator
 from docker.types import DriverConfig, Mount
-from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task
-from util_tasks.t_check_repo import build_check_repo_task
-from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task
-from util_tasks.t_update_docker_image import build_update_image_task
 
 my_email = Variable.get("fail_email")
 addr = Variable.get("repo_server_url")
@@ -59,19 +55,12 @@ def get_random_moll():
 
     sampled_moll = get_random_moll()
 
-    task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name)
-    task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name)
-    task_branch_pull_ssh = build_branch_pull_ssh_task(
-        dag=dag, task_name="alert_inverter_temperature", repo_name=repo_name
-    )
-    task_update_image = build_update_image_task(dag=dag, repo_name=repo_name)
-
     notify_alarms_task = DockerOperator(
         api_version="auto",
         task_id="alert_inverter_temperature",
-        docker_conn_id="somenergia_registry",
-        image="{}/{}-requirements:latest".format(
-            "{{ conn.somenergia_registry.host }}", repo_name
+        docker_conn_id="somenergia_harbor_dades_registry",
+        image="{}/{}-app:latest".format(
+            "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name
         ),
         working_dir=f"/repos/{repo_name}",
         command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\
@@ -83,10 +72,3 @@ def get_random_moll():
         retrieve_output=True,
         trigger_rule="none_failed",
     )
-
-    task_check_repo >> task_git_clone
-    task_check_repo >> task_branch_pull_ssh
-    task_git_clone >> task_update_image
-    task_branch_pull_ssh >> notify_alarms_task
-    task_branch_pull_ssh >> task_update_image
-    task_update_image >> notify_alarms_task
diff --git a/dags/plantmonitor_alert_inverter_zero_power.py b/dags/plantmonitor_alert_inverter_zero_power.py
@@ -5,10 +5,6 @@
 from airflow.models import Variable
 from airflow.providers.docker.operators.docker import DockerOperator
 from docker.types import DriverConfig, Mount
-from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task
-from util_tasks.t_check_repo import build_check_repo_task
-from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task
-from util_tasks.t_update_docker_image import build_update_image_task
 
 my_email = Variable.get("fail_email")
 addr = Variable.get("repo_server_url")
@@ -59,19 +55,12 @@ def get_random_moll():
 
     sampled_moll = get_random_moll()
 
-    task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name)
-    task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name)
-    task_branch_pull_ssh = build_branch_pull_ssh_task(
-        dag=dag, task_name="alert_inverter_zero_power", repo_name=repo_name
-    )
-    task_update_image = build_update_image_task(dag=dag, repo_name=repo_name)
-
     notify_alarms_task = DockerOperator(
         api_version="auto",
         task_id="alert_inverter_zero_power",
-        docker_conn_id="somenergia_registry",
-        image="{}/{}-requirements:latest".format(
-            "{{ conn.somenergia_registry.host }}", repo_name
+        docker_conn_id="somenergia_harbor_dades_registry",
+        image="{}/{}-app:latest".format(
+            "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name
         ),
         working_dir=f"/repos/{repo_name}",
         command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\
@@ -83,10 +72,3 @@ def get_random_moll():
         retrieve_output=True,
         trigger_rule="none_failed",
     )
-
-    task_check_repo >> task_git_clone
-    task_check_repo >> task_branch_pull_ssh
-    task_git_clone >> task_update_image
-    task_branch_pull_ssh >> notify_alarms_task
-    task_branch_pull_ssh >> task_update_image
-    task_update_image >> notify_alarms_task
diff --git a/dags/plantmonitor_alert_meter_no_readings.py b/dags/plantmonitor_alert_meter_no_readings.py
@@ -5,10 +5,6 @@
 from airflow.models import Variable
 from airflow.providers.docker.operators.docker import DockerOperator
 from docker.types import DriverConfig, Mount
-from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task
-from util_tasks.t_check_repo import build_check_repo_task
-from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task
-from util_tasks.t_update_docker_image import build_update_image_task
 
 my_email = Variable.get("fail_email")
 addr = Variable.get("repo_server_url")
@@ -59,19 +55,12 @@ def get_random_moll():
 
     sampled_moll = get_random_moll()
 
-    task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name)
-    task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name)
-    task_branch_pull_ssh = build_branch_pull_ssh_task(
-        dag=dag, task_name="alert_meter_no_readings", repo_name=repo_name
-    )
-    task_update_image = build_update_image_task(dag=dag, repo_name=repo_name)
-
     notify_alarms_task = DockerOperator(
         api_version="auto",
         task_id="alert_meter_no_readings",
-        docker_conn_id="somenergia_registry",
-        image="{}/{}-requirements:latest".format(
-            "{{ conn.somenergia_registry.host }}", repo_name
+        docker_conn_id="somenergia_harbor_dades_registry",
+        image="{}/{}-app:latest".format(
+            "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name
         ),
         working_dir=f"/repos/{repo_name}",
         command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\
@@ -84,9 +73,3 @@ def get_random_moll():
         trigger_rule="none_failed",
     )
 
-    task_check_repo >> task_git_clone
-    task_check_repo >> task_branch_pull_ssh
-    task_git_clone >> task_update_image
-    task_branch_pull_ssh >> notify_alarms_task
-    task_branch_pull_ssh >> task_update_image
-    task_update_image >> notify_alarms_task
diff --git a/dags/plantmonitor_alert_meter_zero_energy.py b/dags/plantmonitor_alert_meter_zero_energy.py
@@ -5,10 +5,6 @@
 from airflow.models import Variable
 from airflow.providers.docker.operators.docker import DockerOperator
 from docker.types import DriverConfig, Mount
-from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task
-from util_tasks.t_check_repo import build_check_repo_task
-from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task
-from util_tasks.t_update_docker_image import build_update_image_task
 
 my_email = Variable.get("fail_email")
 addr = Variable.get("repo_server_url")
@@ -59,19 +55,12 @@ def get_random_moll():
 
     sampled_moll = get_random_moll()
 
-    task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name)
-    task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name)
-    task_branch_pull_ssh = build_branch_pull_ssh_task(
-        dag=dag, task_name="alert_meter_zero_energy", repo_name=repo_name
-    )
-    task_update_image = build_update_image_task(dag=dag, repo_name=repo_name)
-
     notify_alarms_task = DockerOperator(
         api_version="auto",
         task_id="alert_meter_zero_energy",
-        docker_conn_id="somenergia_registry",
-        image="{}/{}-requirements:latest".format(
-            "{{ conn.somenergia_registry.host }}", repo_name
+        docker_conn_id="somenergia_harbor_dades_registry",
+        image="{}/{}-app:latest".format(
+            "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name
         ),
         working_dir=f"/repos/{repo_name}",
         command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\
@@ -84,9 +73,3 @@ def get_random_moll():
         trigger_rule="none_failed",
     )
 
-    task_check_repo >> task_git_clone
-    task_check_repo >> task_branch_pull_ssh
-    task_git_clone >> task_update_image
-    task_branch_pull_ssh >> notify_alarms_task
-    task_branch_pull_ssh >> task_update_image
-    task_update_image >> notify_alarms_task