From a06a3474747336706fb1db18544e087a6cc432fb Mon Sep 17 00:00:00 2001 From: Cory Latschkowski Date: Mon, 18 Nov 2024 15:05:38 -0600 Subject: [PATCH] add: nim --- components/app-configs/nim/README.md | 19 +++++ .../app-configs/nim/base/deployment.yaml | 69 +++++++++++++++++++ .../app-configs/nim/base/kustomization.yaml | 13 ++++ components/app-configs/nim/base/pvc.yaml | 11 +++ components/app-configs/nim/base/route.yaml | 18 +++++ components/app-configs/nim/base/service.yaml | 12 ++++ .../nim/components/gpu/kustomization.yaml | 7 ++ .../nim/components/gpu/patch-deployment.yaml | 12 ++++ .../components/namespace/kustomization.yaml | 5 ++ .../nim/components/namespace/namespace.yaml | 7 ++ .../nim/overlays/default/kustomization.yaml | 5 ++ .../nim/overlays/demo/kustomization.yaml | 11 +++ .../nim/overlays/gpu/kustomization.yaml | 8 +++ .../components/nemo/patch-deployment.yaml | 3 +- 14 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 components/app-configs/nim/README.md create mode 100644 components/app-configs/nim/base/deployment.yaml create mode 100644 components/app-configs/nim/base/kustomization.yaml create mode 100644 components/app-configs/nim/base/pvc.yaml create mode 100644 components/app-configs/nim/base/route.yaml create mode 100644 components/app-configs/nim/base/service.yaml create mode 100644 components/app-configs/nim/components/gpu/kustomization.yaml create mode 100644 components/app-configs/nim/components/gpu/patch-deployment.yaml create mode 100644 components/app-configs/nim/components/namespace/kustomization.yaml create mode 100644 components/app-configs/nim/components/namespace/namespace.yaml create mode 100644 components/app-configs/nim/overlays/default/kustomization.yaml create mode 100644 components/app-configs/nim/overlays/demo/kustomization.yaml create mode 100644 components/app-configs/nim/overlays/gpu/kustomization.yaml diff --git a/components/app-configs/nim/README.md b/components/app-configs/nim/README.md new file mode 100644 index 00000000..64ef6170 --- /dev/null +++ b/components/app-configs/nim/README.md @@ -0,0 +1,19 @@ +# Notes + +## Quickstart + +Query NIM Open AI API + +```sh +URL=https://$(oc get route -o go-template='{{.spec.host}}' nim)/v1/completions + +curl -s -X 'POST' \ + "${URL}" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "meta/llama3-8b-instruct", + "prompt": "Once upon a time", + "max_tokens": 64 + }' | jq .choices[0].text +``` diff --git a/components/app-configs/nim/base/deployment.yaml b/components/app-configs/nim/base/deployment.yaml new file mode 100644 index 00000000..3289d28d --- /dev/null +++ b/components/app-configs/nim/base/deployment.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llama3 +spec: + replicas: 1 + strategy: + type: Recreate + template: + spec: + containers: + - name: nim + image: 'nvcr.io/nim/meta/llama3-8b-instruct:1.0.0' + env: + - name: NGC_API_KEY + value: your-key-here + ports: + - containerPort: 8000 + protocol: TCP + name: api + resources: + limits: + cpu: '2' + memory: 8Gi + requests: + cpu: 500m + memory: 8Gi + # livenessProbe: + # httpGet: + # path: / + # port: http + # scheme: HTTP + # timeoutSeconds: 5 + # periodSeconds: 5 + # successThreshold: 1 + # failureThreshold: 3 + # readinessProbe: + # httpGet: + # path: / + # port: notebook + # scheme: HTTP + # timeoutSeconds: 1 + # periodSeconds: 10 + # successThreshold: 1 + # failureThreshold: 3 + # startupProbe: + # httpGet: + # path: / + # port: notebook + # scheme: HTTP + # timeoutSeconds: 1 + # periodSeconds: 5 + # successThreshold: 1 + # failureThreshold: 10 + volumeMounts: + - name: shm + mountPath: /dev/shm + - name: nim-cache + mountPath: /opt/nim/.cache + subPath: data + volumes: + - name: shm + emptyDir: + medium: Memory + sizeLimit: 512Mi + - name: nim-cache + persistentVolumeClaim: + claimName: nim diff --git a/components/app-configs/nim/base/kustomization.yaml b/components/app-configs/nim/base/kustomization.yaml new file mode 100644 index 00000000..77e846d9 --- /dev/null +++ b/components/app-configs/nim/base/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +labels: +- includeSelectors: true + pairs: + component: nim + +resources: +- deployment.yaml +- pvc.yaml +- route.yaml +- service.yaml diff --git a/components/app-configs/nim/base/pvc.yaml b/components/app-configs/nim/base/pvc.yaml new file mode 100644 index 00000000..aebad04a --- /dev/null +++ b/components/app-configs/nim/base/pvc.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nim +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/components/app-configs/nim/base/route.yaml b/components/app-configs/nim/base/route.yaml new file mode 100644 index 00000000..af2a7fe1 --- /dev/null +++ b/components/app-configs/nim/base/route.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: nim + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + port: + targetPort: 8000 + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + to: + kind: Service + name: api + weight: 100 + wildcardPolicy: None diff --git a/components/app-configs/nim/base/service.yaml b/components/app-configs/nim/base/service.yaml new file mode 100644 index 00000000..a8a6e687 --- /dev/null +++ b/components/app-configs/nim/base/service.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: nim +spec: + ports: + - name: api + port: 8000 + targetPort: 8000 + sessionAffinity: None + type: ClusterIP diff --git a/components/app-configs/nim/components/gpu/kustomization.yaml b/components/app-configs/nim/components/gpu/kustomization.yaml new file mode 100644 index 00000000..02f6c6d8 --- /dev/null +++ b/components/app-configs/nim/components/gpu/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +patches: + - path: patch-deployment.yaml + target: + kind: Deployment diff --git a/components/app-configs/nim/components/gpu/patch-deployment.yaml b/components/app-configs/nim/components/gpu/patch-deployment.yaml new file mode 100644 index 00000000..ec29d012 --- /dev/null +++ b/components/app-configs/nim/components/gpu/patch-deployment.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nim +spec: + template: + spec: + containers: + - name: nim + resources: + limits: + nvidia.com/gpu: 1 diff --git a/components/app-configs/nim/components/namespace/kustomization.yaml b/components/app-configs/nim/components/namespace/kustomization.yaml new file mode 100644 index 00000000..d41e3cf2 --- /dev/null +++ b/components/app-configs/nim/components/namespace/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: +- namespace.yaml diff --git a/components/app-configs/nim/components/namespace/namespace.yaml b/components/app-configs/nim/components/namespace/namespace.yaml new file mode 100644 index 00000000..5b03058b --- /dev/null +++ b/components/app-configs/nim/components/namespace/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: demo-nim + annotations: + openshift.io/display-name: "Demo - NIM" diff --git a/components/app-configs/nim/overlays/default/kustomization.yaml b/components/app-configs/nim/overlays/default/kustomization.yaml new file mode 100644 index 00000000..2b87e6b1 --- /dev/null +++ b/components/app-configs/nim/overlays/default/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base diff --git a/components/app-configs/nim/overlays/demo/kustomization.yaml b/components/app-configs/nim/overlays/demo/kustomization.yaml new file mode 100644 index 00000000..c76b9a61 --- /dev/null +++ b/components/app-configs/nim/overlays/demo/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: demo-nim + +components: +- ../../components/gpu +- ../../components/namespace + +resources: +- ../../base diff --git a/components/app-configs/nim/overlays/gpu/kustomization.yaml b/components/app-configs/nim/overlays/gpu/kustomization.yaml new file mode 100644 index 00000000..ab768395 --- /dev/null +++ b/components/app-configs/nim/overlays/gpu/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: +- ../../components/gpu + +resources: +- ../../base diff --git a/components/app-configs/notebook-standalone/components/nemo/patch-deployment.yaml b/components/app-configs/notebook-standalone/components/nemo/patch-deployment.yaml index db397934..b7986aa1 100644 --- a/components/app-configs/notebook-standalone/components/nemo/patch-deployment.yaml +++ b/components/app-configs/notebook-standalone/components/nemo/patch-deployment.yaml @@ -17,7 +17,8 @@ spec: - | #!/bin/sh jupyter lab \ - --ServerApp.ip=0.0.0.0:8888 \ + --ServerApp.ip=0.0.0.0 \ + --ServerApp.port=8888 \ --ServerApp.allow_origin="*" \ --ServerApp.open_browser=False \ --ServerApp.token=''