Skip to content

Commit

Permalink
fix nginx problem
Browse files Browse the repository at this point in the history
  • Loading branch information
zhanglei1949 committed Oct 11, 2024
1 parent ebe3ab1 commit bd742e6
Show file tree
Hide file tree
Showing 17 changed files with 2,039 additions and 496 deletions.
60 changes: 60 additions & 0 deletions charts/graphscope-interactive/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -133,3 +133,63 @@ export ADMIN_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojso
export QUERY_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[1].ports[0].containerPort}')
```

```bash
# to verify the helm char
helm install lei-test --dry-run
```


## add resty.http to nginx images

A customized nginx image


## nginx conf
<!-- # nginx.conf: |
# events {}
# http {
# server {
# listen 10000;
# server_name localhost;
# location / {
# {{- $baseName := include "graphscope-interactive.secondary.fullname" . }}
# {{- $replicaCount := .Values.backend.replicas }}
# {{- $serviceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.secondary.fullname" .) .Release.Namespace .Values.clusterDomain }}
# {{- $port := .Values.secondary.service.queryPort }}
# proxy_pass http://{{ printf "%s-0.%s:%d" $baseName $serviceName $port }};
# {{- range $i := until $replicaCount }}
# mirror /mirror{{ $i }} {
# internal;
# proxy_pass http://{{ printf "%s-%d.%s:%d%s" $baseName (add $i 1) $serviceName $port $request_uri }};
# }
# {{- end }}
# location /mirror {{ printf "%s-%d.%s:%d" $baseName (add $i 1) $serviceName $port }};
# {{- end }}
# proxy_set_header Host $host;
# proxy_set_header X-Real-IP $remote_addr;
# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# proxy_set_header X-Forwarded-Proto $scheme;
# }
# }
# } -->

# - name: admin-nginx
# image: {{ include "graphscope-interactive.nginx.image" . }}
# imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }}
# # command: ["sleep", "infinity"]
# ports:
# - name: admin-port
# containerPort: {{ .Values.frontend.service.adminPort }}
# {{- if .Values.resources.frontend }}
# resources: {{- toYaml .Values.resources.frontend | nindent 12 }}
# {{- end }}
# volumeMounts:
# - name: workspace
# mountPath: {{ .Values.workspace }}
# - name: config
# mountPath: {{ include "graphscope-interactive.engineConfigPath" . }}
# subPath: engine_config.yaml
# - name: admin-nginx-config
# mountPath: /etc/nginx/nginx.conf
# subPath: nginx.conf
32 changes: 15 additions & 17 deletions charts/graphscope-interactive/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -132,23 +132,6 @@ Return the proper graphscope-interactive secondary image name
{{- end -}}
{{- end -}}

{{/*
Return the proper graphscope-interactive cron job image name
*/}}
{{- define "graphscope-interactive.cronjob.image" -}}
{{- $tag := .Chart.AppVersion | toString -}}
{{- with .Values.cronjob.image -}}
{{- if .tag -}}
{{- $tag = .tag | toString -}}
{{- end -}}
{{- if .registry -}}
{{- printf "%s/%s:%s" .registry .repository $tag -}}
{{- else -}}
{{- printf "%s:%s" .repository $tag -}}
{{- end -}}
{{- end -}}
{{- end -}}


{{/*
Create the name of the service account to use
Expand Down Expand Up @@ -215,6 +198,21 @@ Return the engineConfigPath with the graphscope configuration
/etc/interactive/interactive_config.yaml
{{- end -}}


{{/*
Primary service name.
*/}}
{{- define "graphscope-interactive.primary.serviceName" -}}
{{- printf "%s.%s.svc.%s" (include "graphscope-interactive.primary.fullname" .) .Release.Namespace .Values.clusterDomain }}
{{- end -}}

{{/*
From where a http client could fetch the schema uri
*/}}
{{- define "graphscope-interactive.graphSchemaUri" -}}
{{- printf "http://%s-0.%s:%d" (include "graphscope-interactive.fullname" .) (include "graphscope-interactive.primary.serviceName" .) .Values.primary.service.adminPort | trimSuffix "-" -}}
{{- end -}}

{{/*
Return the realEngineConfigPath with the graphscope configuration, templated by frontend
*/}}
Expand Down
159 changes: 159 additions & 0 deletions charts/graphscope-interactive/templates/admin_nginx_conf.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Release.Name }}-admin-nginx-config
namespace: {{ .Release.Namespace }}
labels: {{- include "graphscope-interactive.labels" . | nindent 4 }}
app.kubernetes.io/component: configmap
{{- if .Values.commonLabels }}
{{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- end }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
data:
nginx.conf: |
events {}
http {
resolver local=on valid=5s;
server {
{{- $adminPort := .Values.primary.service.adminPort | int }}
listen {{ $adminPort }};
client_body_buffer_size 10M;
client_max_body_size 10M;
location / {
{{- $primaryBaseName := include "graphscope-interactive.primary.fullname" . }}
{{- $secondaryBaseName := include "graphscope-interactive.secondary.fullname" . }}
{{- $replicaCount := .Values.secondary.replicaCount | int }}
{{- $primaryServiceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.primary.fullname" .) .Release.Namespace .Values.clusterDomain }}
{{- $secondaryServiceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.secondary.fullname" .) .Release.Namespace .Values.clusterDomain }}
{{- $port := .Values.secondary.service.adminPort | int }}
proxy_pass {{ printf "http://%s-0.%s:%d" $primaryBaseName $primaryServiceName $port | quote }};
content_by_lua_block {
function arrayToString(arr, separator)
separator = separator or ", " -- Default separator if not provided
return table.concat(arr, separator)
end
function send_request(http, full_uri, method, body_data, headers)
local httpc = http.new()
if method == "GET" or method == "DELETE" then
return httpc:request_uri(full_uri, {
method = method,
})
elseif method == "POST" or method == "PUT" then
return httpc:request_uri(full_uri, {
method = method,
body = body_data,
headers = headers
})
else
ngx.log(ngx.ERR, " not recognized method ", method)
end
end
local http = require "resty.http"
local res = {}
local status_codes = {}
local error_message = nil -- Initialize a variable to capture error messages
local urls = {
{{ printf "http://%s-0.%s:%d" $primaryBaseName $primaryServiceName $port | quote }};
{{- if eq $replicaCount 1 }}
{{ printf "http://%s-0.%s:%d" $secondaryBaseName $secondaryServiceName $port | quote }}
{{- else }}
{{- range $i := until (sub $replicaCount 1 | int ) }}
{{ printf "\"http://%s-%d.%s:%d\"," $secondaryBaseName $i $secondaryServiceName $port }}
{{- end }}
{{ printf "http://%s-%d.%s:%d" $secondaryBaseName (sub $replicaCount 1) $secondaryServiceName $port | quote }}
{{- end }}
}
local original_headers = ngx.req.get_headers()
local request_uri=ngx.var.request_uri
local method = ngx.req.get_method()
-- Create a table for modified headers
local backend_headers = {}
-- Copy the relevant headers, if needed, or modify them
for key, value in pairs(original_headers) do
-- You can filter headers if needed (e.g., skip "host" or "authorization")
if key ~= "Host" and key ~= "User-Agent" and key ~= "Content-Length" then
backend_headers[key] = value
end
end
ngx.req.read_body() -- Read the request body
-- resize status_codes to the number of replicas
for i = 1, #urls do
status_codes[i] = 0
res[i] = ""
end
local threads = {}
local body_data = ngx.req.get_body_data()
for index, backend in ipairs(urls) do
-- full_uri is backend + request_uri
local full_uri = backend .. request_uri
threads[index] = ngx.thread.spawn(function()
local response, err = send_request(http, full_uri, method, body_data, backend_headers)
local status_code = 0
if response ~= nil then
status_code = response.status
res[index] = response.body
if response.status < 200 or response.status >= 300 then
if not error_message then -- Capture the error message from the first failed request
error_message = response.body or "Failed request without a body."
end
end
else
status_code = 500
if err ~= nil then
ngx.log(ngx.ERR, "Failed to request: ", err)
if not error_message then -- Capture error when no response
error_message = "Error: " .. err
end
else
error_message = "Not found"
end
end
status_codes[index] = status_code
end)
end
for _, thread in ipairs(threads) do
coroutine.resume(thread)
end
for _, thread in ipairs(threads) do
ngx.thread.wait(thread)
end
local success = true
local final_status_code = 200
for i = 1, #urls do
if status_codes[i] < 200 or status_codes[i] >= 300 then
ngx.log(ngx.ERR, "Failed to request: ", urls[i], " with status code: ", status_codes[i], " and response: ", res[i], " index: ", i)
success = false
final_status_code = status_codes[i]
break
end
end
ngx.header.content_type = 'application/json'
if success then
ngx.status = final_status_code
ngx.log(ngx.INFO, "Success: ", arrayToString(res, ", "), " with status code: ", final_status_code)
ngx.say(res[1])
ngx.exit(final_status_code)
else
ngx.status = final_status_code
ngx.log(ngx.ERR, "Failed to request: ", error_message, " with status code: ", final_status_code)
ngx.say(error_message)
ngx.exit(final_status_code)
end
}
}
}
}
2 changes: 1 addition & 1 deletion charts/graphscope-interactive/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ metadata:
{{- end }}
data:
interactive_config.yaml: |-
log_level: {{ .Values.engine.logLevel }}
log_level: {{ .Values.logLevel }}
default_graph: {{ .Values.defaultGraph }}
compute_engine:
type: hiactor
Expand Down
Loading

0 comments on commit bd742e6

Please sign in to comment.