chore(infra): restart container when action runs (#46)

* chore: enable flush on shutdown for loki * refactor: remove unused code * chore: restart containers when action runs
skkuding · Apr 2, 2024 · 7b788cf · 7b788cf
1 parent 87d997f
commit 7b788cf
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 33 deletions.
diff --git a/.github/workflows/update-stage.yml b/.github/workflows/update-stage.yml
@@ -23,31 +23,32 @@ jobs:
           MINIO_ROOT_PASSWORD = ${{ secrets.MINIO_ROOT_PASSWORD }}
           EOF
 
-      - name: Check if Caddy containers are running
-        id: check-caddy-container
+      - name: Check if initial containers are running
+        id: check-start-up-container
         run: |
           {
             echo 'stdout<<EOF'
-            docker compose --profile caddy ps -q
+            docker compose --profile start-up ps -q
             echo EOF
           } >> "$GITHUB_OUTPUT"
           
-      - name: when caddy container down, caddy up
-        if: steps.check-caddy-container.outputs.stdout == ''
-        run: |
-          docker compose --profile caddy up -d
+      - name: when initial containers are not running, start all containers
+        if: steps.check-start-up-container.outputs.stdout == ''
+        run: >
+          docker compose --profile start-up --profile log 
+          --profile trace --profile metric up -d
 
-      - name: Run Docker Compose for Log (Loki, Minio, Grafana)
+      - name: Restart containers for Log (Loki, Minio)
         run: |
-          docker compose --profile log up -d
+          docker compose --profile log restart
 
-      - name: Run Docker Compose Of Trace (Agent, Tempo, Grafana)
+      - name: Restart containers Of Trace (Tempo, Minio)
         run: |
-          docker compose --profile trace up -d 
+          docker compose --profile trace restart
 
-      - name: Run Docker Compose for Metric (Agent, Prometheus, Grafana)
+      - name: Restart containers for Metric (Prometheus)
         run: |
-          docker compose --profile metric up -d 
+          docker compose --profile metric restart
 
       - name: Set Caddyfile Environment Variables
         run: |

diff --git a/config/loki/loki-config-aws.yml b/config/loki/loki-config-aws.yml
@@ -29,6 +29,8 @@ ingester:
   chunk_target_size: 1048576 # Target size of 1MB before flushing.
   chunk_retain_period: 30s # Retain chunks for 30 seconds in memory after flushing.
   max_transfer_retries: 0
+  wal:
+    flush_on_shutdown: true
 
 storage_config:
   aws:

diff --git a/config/loki/loki-config-stage.yml b/config/loki/loki-config-stage.yml
@@ -29,6 +29,8 @@ ingester:
   chunk_target_size: 1048576 # Target size of 1MB before flushing.
   chunk_retain_period: 30s # Retain chunks for 30 seconds in memory after flushing.
   max_transfer_retries: 0
+  wal:
+    flush_on_shutdown: true
 
 storage_config:
   aws:

diff --git a/config/loki/loki-config-test.yml b/config/loki/loki-config-test.yml
@@ -29,6 +29,8 @@ ingester:
   chunk_target_size: 1048576 # Target size of 1MB before flushing.
   chunk_retain_period: 30s # Retain chunks for 30 seconds in memory after flushing.
   max_transfer_retries: 0
+  wal:
+    flush_on_shutdown: true
 
 storage_config:
   aws:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   caddy:
-    profiles: ["caddy"]
+    profiles: ["start-up"]
     image: caddy:2.7.6-alpine
     container_name: caddy
     restart: always
@@ -12,7 +12,7 @@ services:
     network_mode: host
 
   minio:
-    profiles: ["log", "trace"]
+    profiles: ["start-up"] 
     container_name: minio
     image: minio/minio:latest
     volumes:
@@ -27,7 +27,7 @@ services:
     shm_size: "1gb"
 
   createbuckets:
-    profiles: ["log", "trace"]
+    profiles: ["start-up", "log", "trace"]
     image: minio/mc
     volumes:
       - minio_data_volume:/data
@@ -59,6 +59,10 @@ services:
     volumes:
       - "$PWD/config/loki/loki-config-test.yml:/etc/loki/loki-config.yml"
     command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
+    depends_on:
+      - minio
+    restart: always
+
 
   loki-stage:
     profiles: ["log"]
@@ -72,6 +76,9 @@ services:
     volumes:
       - "$PWD/config/loki/loki-config-stage.yml:/etc/loki/loki-config.yml"
     command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
+    depends_on:
+      - minio
+    restart: always
 
   loki-aws:
     profiles: ["log"]
@@ -85,6 +92,9 @@ services:
     volumes:
       - "$PWD/config/loki/loki-config-aws.yml:/etc/loki/loki-config.yml"
     command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true
+    depends_on:
+      - minio
+    restart: always
 
   prometheus:
     profiles: ["metric"]
@@ -99,6 +109,7 @@ services:
       - '--config.file=/etc/prometheus/prometheus-config.yml'
       - '--storage.tsdb.path=/prometheus'
       - '--web.enable-remote-write-receiver'
+    restart: always
 
   tempo:
     profiles: ["trace"]
@@ -118,31 +129,16 @@ services:
       - "-config.expand-env=true" 
     depends_on:
       - minio
-
-  # agent:
-  #   profiles: ["metric", "trace"]
-  #   image: grafana/agent:latest
-  #   container_name: agent
-  #   environment:
-  #     - AGENT_MODE=flow
-  #   ports:
-  #     - "12345:12345"
-  #     - "4318:4318"   # OTLP over HTTP receiver
-  #   volumes: 
-  #     - $PWD/config/agent/config.river:/etc/agent/config.river
-  #   command: run --server.http.listen-addr=0.0.0.0:12345 /etc/agent/config.river
+    restart: always
 
   grafana:
-    profiles: ["log", "metric", "trace"]
+    profiles: ["start-up"]
     image: grafana/grafana:latest
     env_file:
       - .env
     container_name: grafana
     restart:
       unless-stopped
-      #    environment:
-      #      - TERM=linux
-      #      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-polystat-panel
     ports:
       - "3000:3000"
     volumes: