Karajendran rag

use-cases/rag-pipeline/README.md

@@ -1,60 +1,17 @@
 # Steps
 Run following commands in cloud shell
 
-# Pre Req
+## Pre Req
 source $MLP_PLATFORM_ENV_FILE
 gcloud config set project $MLP_PROJECT_ID
 
-## Create Google Service Account
-gcloud iam service-accounts create alloydb-access-sa \
-  --display-name="AlloyDB Admin Service Account"
+## Setup the platform:
+Follow Readme under accelerated-platforms/platforms folder
+It will create AlloyDB instance and GKE cluster and enable PSC
 
-## Add following roles:
-
-gcloud projects add-iam-policy-binding $MLP_PROJECT_ID \
-  --member="serviceAccount:alloydb-access-sa@$MLP_PROJECT_ID.iam.gserviceaccount.com" \
-  --role="roles/alloydb.admin" \
-  --condition="None"
-
-gcloud projects add-iam-policy-binding $MLP_PROJECT_ID \
-  --member=serviceAccount:alloydb-access-sa@$MLP_PROJECT_ID.iam.gserviceaccount.com --role="roles/alloydb.client" \
-  --condition="None"
-
-gcloud projects add-iam-policy-binding $MLP_PROJECT_ID \
-  --member=serviceAccount:alloydb-access-sa@$MLP_PROJECT_ID.iam.gserviceaccount.com --role="roles/serviceusage.serviceUsageConsumer" \
-  --condition="None"
-
--- Add Storage Admin
-gcloud projects add-iam-policy-binding $MLP_PROJECT_ID --member=serviceAccount:alloydb-access-sa@$MLP_PROJECT_ID.iam.gserviceaccount.com --role="roles/storage.objectAdmin" --condition="None"
-
-gcloud iam service-accounts add-iam-policy-binding \
-    [email protected] \
-    --member="serviceAccount:gkebatchexpce3c8dcb.svc.id.goog[ml-team/rag-ksa]" \
-    --role="roles/iam.serviceAccountTokenCreator" \
-    --condition="None"
-
-
-# Workload Identity Creation
-## 1. Get fleet credentials
+## Get Fleet credentials
 gcloud container fleet memberships get-credentials ${MLP_CLUSTER_NAME} --project ${MLP_PROJECT_ID}
 
-## 2. Create a Kubernetes service account in your GKE cluster
-kubectl create serviceaccount rag-ksa -n ml-team
-
-## 3. Annotate the Kubernetes service account with the email address of the Google Cloud service account
-kubectl annotate serviceaccount rag-ksa \
-  iam.gke.io/gcp-service-account=alloydb-access-sa@$MLP_PROJECT_ID.iam.gserviceaccount.com
-  -n ml-team
-
-
-## 4. Bind the Google Cloud service account to the Kubernetes service account using Workload Identity
-
-gcloud iam service-accounts add-iam-policy-binding \
-   --role="roles/iam.workloadIdentityUser" \
-   --member="serviceAccount:gkebatchexpce3c8dcb.svc.id.goog[ml-team/rag-ksa]" \
-   [email protected] \
-   --condition="None"
-
 # Deploy RAG Architecture
 1. Deploy Multi modal embedding model in GKE
 - cd multimodal-emb/
@@ -63,7 +20,7 @@ gcloud iam service-accounts add-iam-policy-binding \
 2. Build AlloyDB
 - cd alloy-db/
 - Follow README.md:
-    - Creates AlloyDB cluster, instance, users, database, table; Populate the product catalog and generate embeddings, vector index on text_embeddings
+    - Creates Database, Table; Populates the product catalog with embeddings; Create vector index on text_embeddings.
 
 3. Deploy instruction-tuned model in GKE
 - cd instruction-tuned-model-deployment/
@@ -72,3 +29,6 @@ gcloud iam service-accounts add-iam-policy-binding \
 4. Deploy backend application in GKE
 - cd backend-application
 - Follow README.md 
+
+5. Test with curl job in backend-application
+   kubectl apply -f curl-job.yaml -n ml-team

use-cases/rag-pipeline/alloy-db/README.md

@@ -16,29 +16,13 @@ docker tag catalog-onboarding:latest gcr.io/$MLP_PROJECT_ID/alloydb-setup:latest
 docker push gcr.io/$MLP_PROJECT_ID/alloydb-setup:latest
 
 
-3. TODO: Revisit this step
+3. Use KSA in the job and Apply the Job to your GKE cluster:
 
-kubectl create secret generic alloydb-secrets \
-  --from-literal=project_id=$MLP_PROJECT_ID \
-  --from-literal=password=[YOUR_PASSWORD] \
-  -n ml-team
-
-Create a Kubernetes Secret for sensitive data.
-
-Bash
-kubectl create secret generic alloydb-secrets \
-  --from-literal=project_id=$MLP_PROJECT_ID \
-  --from-literal=password=[YOUR_PASSWORD] \
-  --from-literal=catalog-admin-password=[CATALOG_ADMIN_PASSWORD] \
-  --from-literal=rag-user-password=[RAG_USER_PASSWORD]
-  -n ml-team
-
-
-# 6. Use KSA in the job and Apply the Job to your GKE cluster:
+gcloud container fleet memberships get-credentials ${MLP_CLUSTER_NAME} --project ${MLP_PROJECT_ID}
 
 Bash
 kubectl apply -f  alloydb-setup-job.yaml -n ml-team 
 
 Check logs:
 kubectl get pods -n ml-team
-kubectl logs -f catalog-onboarding-job-X -n ml-team
+kubectl logs -f alloydb-setup-xxxxx -n ml-team

use-cases/rag-pipeline/alloy-db/manifests/alloydb-setup-job.yaml

@@ -1,29 +1,49 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: alloydb-setup-job
+  name: alloydb-setup
 spec:
   template:
     spec:
-      serviceAccountName: rag-ksa # Service account with necessary permissions
+      serviceAccountName: dev-rag-ml-team-db-admin
       containers:
       - name: alloydb-setup
-        image: gcr.io/gkebatchexpce3c8dcb/alloydb-setup:latest # IMAGE_URL
+        image: gcr.io/gkebatchexpce3c8dcb/alloydb-setup:latest
         env:
-        - name: PROJECT_ID
-          #valueFrom:
-            #secretKeyRef:
-              #name: alloydb-secrets
-              #key: project_id
-        - name: REGION
-          value: "us-central1"
-        - name: CLUSTER_ID
-          value: "my-cluster"
-        - name: INSTANCE_ID
-          value: "my-instance"
-        - name: PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: alloydb-secrets
-              key: password
+        - name: PROCESSED_DATA_BUCKET
+          value: "gkebatchexpce3c8dcb-dev-rag-data"
+        - name: MASTER_CATALOG_FILE_NAME
+          value: "RAG/master_product_catalog.csv"
+        - name: CATALOG_DB_NAME
+          value: "product_catalog"
+        - name: CATALOG_TABLE_NAME
+          value: "clothes"
+        - name: MLP_DB_ADMIN_IAM
+          value: "[email protected]"
+        - name: EMBEDDING_DIMENSION
+          value: "768"
+        - name: EMBEDDING_COLUMN
+          value: "text_embeddings"
+        - name: NUM_LEAVES_VALUE
+          value: "300"
+        - name: MLP_DB_INSTANCE_URI
+          value: "projects/gkebatchexpce3c8dcb/locations/us-central1/clusters/mlp-dev-rag/instances/mlp-dev-rag-primary"
+        - name: EMBEDDING_MODEL
+          value: "multimodal-embedding-a100"
+        - name: MLP_KUBERNETES_NAMESPACE
+          value: "ml-team"
       restartPolicy: Never

use-cases/rag-pipeline/alloy-db/src/Dockerfile

@@ -1,16 +1,23 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM python:3.13.0-slim
 
-WORKDIR /app
+WORKDIR /workspace
 
-COPY requirements.txt requirements.txt
-RUN pip install --no-cache-dir -r requirements.txt
+COPY requirements.txt alloydb_setup.py create_catalog.py custom_json_formatter.py get_emb.py logging.conf main.py /workspace/
 
-COPY alloydb_setup.py .
-COPY create_catalog.py .
-COPY get_emb.py .
-COPY main.py .
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
 
-#CMD ["python", "alloydb_setup.py"]
-#CMD ["python", "create_catalog.py"]
 CMD ["python", "main.py"]
-