Skip to content

Commit

Permalink
full refactor for more efficient ressource management with prefect wo…
Browse files Browse the repository at this point in the history
…rkers. Dockerfiles now only boot up when flows are executed. Massively saving necessaary dedicated

runtime compute
  • Loading branch information
JimVincentW committed Dec 31, 2024
1 parent 2236bb5 commit 9513fac
Show file tree
Hide file tree
Showing 428 changed files with 1,538 additions and 2,353 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ spec:
ports:
- containerPort: 5432
args:
- "--credentials-file=/credentials/postgres-service-key.json"
- "--credentials-file=/credentials/service-postgres-key.json"
- "{{ .Values.cloudSqlProxy.connectionName }}"
volumeMounts:
- name: cloud-sql-proxy-credentials
Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: embedding-service
name: service-embeddings
namespace: {{ .Values.namespace }}
spec:
replicas: {{ .Values.replicaCount.embeddingService }}
selector:
matchLabels:
app: embedding-service
app: service-embeddings
template:
metadata:
labels:
app: embedding-service
app: service-embeddings
spec:
containers:
- name: embedding-service
- name: service-embeddings
image: {{ .Values.image.embeddingService }}
ports:
- containerPort: 420
Expand All @@ -35,4 +35,4 @@ spec:
value: "{{ .Values.env.config.PREFECT_API_URL }}"
# Add other environment variables as needed
dependsOn:
- postgres-service
- service-postgres
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: entity-service
name: service-entities
namespace: {{ .Values.namespace }}
spec:
replicas: {{ .Values.replicaCount.entityService }}
selector:
matchLabels:
app: entity-service
app: service-entities
template:
metadata:
labels:
app: entity-service
app: service-entities
spec:
containers:
- name: entity-service
- name: service-entities
image: {{ .Values.image.entityService }}
ports:
- containerPort: 1290
Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: geo-service
name: service-geo
namespace: {{ .Values.namespace }}
spec:
replicas: {{ .Values.replicaCount.geoService }}
selector:
matchLabels:
app: geo-service
app: service-geo
template:
metadata:
labels:
app: geo-service
app: service-geo
spec:
containers:
- name: geo-service
- name: service-geo
image: {{ .Values.image.geoService }}
ports:
- containerPort: 3690
Expand All @@ -26,7 +26,7 @@ spec:
expose:
- 3690
volumeMounts:
- name: geo-service-volume
- name: service-geo-volume
mountPath: /app/data
resources:
requests:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgres-service
name: service-postgres
namespace: {{ .Values.namespace }}
spec:
replicas: {{ .Values.replicaCount.postgresService }}
selector:
matchLabels:
app: postgres-service
app: service-postgres
template:
metadata:
labels:
app: postgres-service
app: service-postgres
spec:
containers:
- name: postgres-service
- name: service-postgres
image: {{ .Values.image.postgresService }}
ports:
- containerPort: 5432
Expand Down Expand Up @@ -44,4 +44,4 @@ spec:
volumes:
- name: postgres-storage
persistentVolumeClaim:
claimName: postgres-service-pvc
claimName: service-postgres-pvc
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: scraper-service
name: service-scraper
namespace: {{ .Values.namespace }}
spec:
replicas: {{ .Values.replicaCount.scraperService }}
selector:
matchLabels:
app: scraper-service
app: service-scraper
template:
metadata:
labels:
app: scraper-service
app: service-scraper
spec:
containers:
- name: scraper-service
- name: service-scraper
image: {{ .Values.image.scraperService }}
ports:
- containerPort: 8081
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: postgres-service-pvc
name: service-postgres-pvc
namespace: {{ .Values.namespace }}
spec:
accessModes:
Expand Down Expand Up @@ -41,7 +41,7 @@ spec:
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: scraper-service-pvc
name: service-scraper-pvc
namespace: {{ .Values.namespace }}
spec:
accessModes:
Expand Down
4 changes: 2 additions & 2 deletions .deployment/opol-cluster-chart/templates/secrets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ spec:
pathType: Prefix
backend:
service:
name: {{ .Release.Name }}-entity-service
name: {{ .Release.Name }}-service-entities
port:
number: 80
- path: /api/geo
pathType: Prefix
backend:
service:
name: {{ .Release.Name }}-geo-service
name: {{ .Release.Name }}-service-geo
port:
number: 80
- path: /prefect
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
apiVersion: v1
kind: Service
metadata:
name: embedding-service
name: service-embeddings
namespace: {{ .Values.namespace }}
spec:
type: {{ .Values.service.embeddingService.type }}
ports:
- port: {{ .Values.service.embeddingService.port }}
targetPort: {{ .Values.service.embeddingService.targetPort }}
selector:
app: embedding-service
app: service-embeddings
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
apiVersion: v1
kind: Service
metadata:
name: entity-service
name: service-entities
namespace: {{ .Values.namespace }}
spec:
type: {{ .Values.service.entityService.type }}
ports:
- port: {{ .Values.service.entityService.port }}
targetPort: {{ .Values.service.entityService.targetPort }}
selector:
app: entity-service
app: service-entities
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
apiVersion: v1
kind: Service
metadata:
name: geo-service
name: service-geo
namespace: {{ .Values.namespace }}
spec:
type: {{ .Values.service.geoService.type }}
ports:
- port: {{ .Values.service.geoService.port }}
targetPort: {{ .Values.service.geoService.targetPort }}
selector:
app: geo-service
app: service-geo
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
apiVersion: v1
kind: Service
metadata:
name: postgres-service
name: service-postgres
namespace: {{ .Values.namespace }}
spec:
type: {{ .Values.service.postgresService.type }}
ports:
- port: {{ .Values.service.postgresService.port }}
targetPort: {{ .Values.service.postgresService.targetPort }}
selector:
app: postgres-service
app: service-postgres
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
apiVersion: v1
kind: Service
metadata:
name: scraper-service
name: service-scraper
namespace: {{ .Values.namespace }}
spec:
type: {{ .Values.service.scraperService.type }}
ports:
- port: {{ .Values.service.scraperService.port }}
targetPort: {{ .Values.service.scraperService.targetPort }}
selector:
app: scraper-service
app: service-scraper
16 changes: 8 additions & 8 deletions .deployment/opol-cluster-chart/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -146,17 +146,17 @@ ingress:
pathType: Prefix
appName: classification-service
servicePort: 5688
- path: /embedding-service
- path: /service-embeddings
pathType: Prefix
appName: embedding-service
appName: service-embeddings
servicePort: 420
- path: /entity-service
- path: /service-entities
pathType: Prefix
appName: entity-service
appName: service-entities
servicePort: 1290
- path: /geo-service
- path: /service-geo
pathType: Prefix
appName: geo-service
appName: service-geo
servicePort: 3690
- path: /litellm
pathType: Prefix
Expand All @@ -174,9 +174,9 @@ ingress:
pathType: Prefix
appName: pelias-placeholder
servicePort: 4434
- path: /scraper-service
- path: /service-scraper
pathType: Prefix
appName: scraper-service
appName: service-scraper
servicePort: 8081
tls:
- hosts:
Expand Down
Loading

0 comments on commit 9513fac

Please sign in to comment.