Merge pull request #9 from tryolabs/feature/add-model-gcs

Added bucket for model logic

.github/workflows/cd.yml

@@ -1,7 +1,7 @@
 name: 'Continuous Delivery'
 
 on:
-  pull_request: #TODO: For testing
+  pull_request:
     branches:
       - "main"
 
@@ -26,11 +26,15 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt -r requirements-dev.txt -r requirements-test.txt
 
-      # Run training script to have the model available
-      - name: Run training script
+      # Download model from GCS
+      - name: Download model from GCS
+        env:
+            BUCKET_NAME: ${{ secrets.BUCKET_NAME }}
+            MODEL: ${{ secrets.MODEL }}
         run: |
           export PYTHONPATH=$PYTHONPATH:$(pwd)
-          python ./challenge/training.py
+          mkdir -p ./challenge/models
+          gsutil cp gs://$BUCKET_NAME/$MODEL.pkl ./challenge/models/$MODEL.pkl
 
       # Build and push Docker image
       - name: Build and push Docker image

.github/workflows/ci.yml

@@ -33,6 +33,13 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt -r requirements-dev.txt -r requirements-test.txt
+
+      # Authenticate to Google Cloud
+      - id: 'auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
 
       # Run tests
       - name: Run tests

challenge/model.py

@@ -7,12 +7,13 @@
 import pandas as pd
 import numpy as np
 import xgboost as xgb
+from google.cloud import storage
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import confusion_matrix, classification_report
 
 MODEL_PATH = os.path.join('challenge', 'models', 'model.pkl')
 THRESHOLD_IN_MINUTES = 15
-
+BUKET_NAME = 'latam-model-challenge'
 class DelayModel:
 
     def __init__(
@@ -44,6 +45,15 @@ def get_min_diff(self, data):
         fecha_i = datetime.strptime(data['Fecha-I'], '%Y-%m-%d %H:%M:%S')
         min_diff = ((fecha_o - fecha_i).total_seconds())/60
         return min_diff
+
+    def _upload_to_gcs(self, source_file_name, destination_blob_name):
+        """Uploads a file to the bucket."""
+        storage_client = storage.Client()
+        bucket = storage_client.bucket(BUKET_NAME)
+        blob = bucket.blob(destination_blob_name)
+        blob.upload_from_filename(source_file_name)
+
+        logging.info(f"File {source_file_name} uploaded to {destination_blob_name}.")
 
     def preprocess(
         self,
@@ -123,6 +133,9 @@ def fit(
         # Save the model to a file
         with open(MODEL_PATH, 'wb') as f:
             pickle.dump(self._model, f)
+
+        # Upload the model to Google Cloud Storage
+        self._upload_to_gcs(MODEL_PATH, os.path.basename(MODEL_PATH))
 
     def predict(
         self,

challenge/terraform/main.tf

@@ -25,6 +25,11 @@ resource "google_cloud_run_service" "latam_challenge_service" {
   }
 }
 
+resource "google_storage_bucket" "bucket" {
+  name     = var.bucket_name
+  location = var.region
+}
+
 resource "google_cloud_run_service_iam_policy" "noauth" {
   location    = google_cloud_run_service.latam_challenge_service.location
   project     = google_cloud_run_service.latam_challenge_service.project

challenge/terraform/terraform.tfstate

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.5.2",
-  "serial": 11,
+  "serial": 16,
   "lineage": "41caa1a9-fd55-f2c3-6cf1-50daff5a262d",
   "outputs": {
     "service_url": {
@@ -52,19 +52,21 @@
               {
                 "annotations": {},
                 "effective_annotations": {
+                  "run.googleapis.com/client-name": "gcloud",
+                  "run.googleapis.com/client-version": "481.0.0",
                   "run.googleapis.com/ingress": "all",
                   "run.googleapis.com/ingress-status": "all",
-                  "run.googleapis.com/operation-id": "798e9ce8-f9a8-4c27-a595-6ab85ec73a63",
+                  "run.googleapis.com/operation-id": "b406ad63-11e9-4d32-ab22-7dd6ff009f67",
                   "serving.knative.dev/creator": "[email protected]",
                   "serving.knative.dev/lastModifier": "[email protected]"
                 },
                 "effective_labels": {
                   "cloud.googleapis.com/location": "us-central1"
                 },
-                "generation": 1,
+                "generation": 6,
                 "labels": {},
                 "namespace": "florencia-tryolabs-latam",
-                "resource_version": "AAYbzoRf3lE",
+                "resource_version": "AAYb5Qwfytc",
                 "self_link": "/apis/serving.knative.dev/v1/namespaces/307648342667/services/latam-challenge",
                 "terraform_labels": {},
                 "uid": "22637102-e0fa-49ea-8643-85dfc7604be4"
@@ -94,14 +96,14 @@
                     "type": "RoutesReady"
                   }
                 ],
-                "latest_created_revision_name": "latam-challenge-00001-xqz",
-                "latest_ready_revision_name": "latam-challenge-00001-xqz",
-                "observed_generation": 1,
+                "latest_created_revision_name": "latam-challenge-00006-6hb",
+                "latest_ready_revision_name": "latam-challenge-00006-6hb",
+                "observed_generation": 6,
                 "traffic": [
                   {
                     "latest_revision": true,
                     "percent": 100,
-                    "revision_name": "latam-challenge-00001-xqz",
+                    "revision_name": "latam-challenge-00006-6hb",
                     "tag": "",
                     "url": ""
                   }
@@ -114,10 +116,13 @@
                 "metadata": [
                   {
                     "annotations": {
-                      "autoscaling.knative.dev/maxScale": "100"
+                      "autoscaling.knative.dev/maxScale": "100",
+                      "run.googleapis.com/client-name": "gcloud",
+                      "run.googleapis.com/client-version": "481.0.0"
                     },
                     "generation": 0,
                     "labels": {
+                      "client.knative.dev/nonce": "nmztqcazkn",
                       "run.googleapis.com/startupProbeType": "Default"
                     },
                     "name": "",
@@ -132,8 +137,8 @@
                     "container_concurrency": 80,
                     "containers": [
                       {
-                        "args": null,
-                        "command": null,
+                        "args": [],
+                        "command": [],
                         "env": [],
                         "env_from": [],
                         "image": "us-docker.pkg.dev/florencia-tryolabs-latam/florencia-repo-latam-challenge/latam-challenge",
@@ -222,6 +227,55 @@
           ]
         }
       ]
+    },
+    {
+      "mode": "managed",
+      "type": "google_storage_bucket",
+      "name": "bucket",
+      "provider": "provider[\"registry.terraform.io/hashicorp/google\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "autoclass": [],
+            "cors": [],
+            "custom_placement_config": [],
+            "default_event_based_hold": false,
+            "effective_labels": {},
+            "enable_object_retention": false,
+            "encryption": [],
+            "force_destroy": false,
+            "id": "latam-model-challenge",
+            "labels": {},
+            "lifecycle_rule": [],
+            "location": "US-CENTRAL1",
+            "logging": [],
+            "name": "latam-model-challenge",
+            "project": "florencia-tryolabs-latam",
+            "project_number": 307648342667,
+            "public_access_prevention": "inherited",
+            "requester_pays": false,
+            "retention_policy": [],
+            "rpo": null,
+            "self_link": "https://www.googleapis.com/storage/v1/b/latam-model-challenge",
+            "soft_delete_policy": [
+              {
+                "effective_time": "2024-06-27T20:33:31.173Z",
+                "retention_duration_seconds": 604800
+              }
+            ],
+            "storage_class": "STANDARD",
+            "terraform_labels": {},
+            "timeouts": null,
+            "uniform_bucket_level_access": false,
+            "url": "gs://latam-model-challenge",
+            "versioning": [],
+            "website": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsInJlYWQiOjI0MDAwMDAwMDAwMCwidXBkYXRlIjoyNDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
+        }
+      ]
     }
   ],
   "check_results": null