ssh-tunnel fix and http connections reuse (#330)

Signed-off-by: Diego Ciangottini <[email protected]>
interTwin-eu · Nov 19, 2024 · 8384292 · 8384292
1 parent 3349a27
commit 8384292
Show file tree

Hide file tree

Showing 18 changed files with 244 additions and 220 deletions.
diff --git a/README.md b/README.md
@@ -1,12 +1,11 @@
-![GitHub License](https://img.shields.io/github/license/intertwin-eu/interlink)
-![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/intertwin-eu/interlink/total)
+[![GitHub License](https://img.shields.io/github/license/intertwin-eu/interlink)](https://img.shields.io/github/license/intertwin-eu/interlink)
 ![GitHub Repo stars](https://img.shields.io/github/stars/intertwin-eu/interlink)
 
 ![GitHub Release](https://img.shields.io/github/v/release/intertwin-eu/interlink)
 ![Tested with Dagger](https://img.shields.io/badge/tested_with_dagger-v0.13.3-green)
+[![Go Report Card](https://goreportcard.com/badge/github.com/intertwin-eu/interlink)](https://goreportcard.com/report/github.com/intertwin-eu/interlink)
 
-![Home Page](https://img.shields.io/badge/home_page-orange?link=https%3A%2F%2Fintertwin-eu.github.io%2Ft%2FinterLink%2F)
-![Slack server](https://img.shields.io/badge/slack_server-8A2BE2?link=https%3A%2F%2Fjoin.slack.com%2Ft%2Fintertwin%2Fshared_invite%2Fzt-2cs67h9wz-2DFQ6EiSQGS1vlbbbJHctA)
+[![Slack server](https://img.shields.io/badge/slack_server-8A2BE2?link=https%3A%2F%2Fjoin.slack.com%2Ft%2Fintertwin%2Fshared_invite%2Fzt-2cs67h9wz-2DFQ6EiSQGS1vlbbbJHctA)](https://join.slack.com/t/intertwin/shared_invite/zt-2cs67h9wz-2DFQ6EiSQGS1vlbbbJHctA)
 
 ![Interlink logo](./docs/static/img/interlink_logo.png)
 

diff --git a/ci/main.go b/ci/main.go
@@ -56,6 +56,7 @@ type Interlink struct {
 	KubeConfigHost     *dagger.File
 	InterlinkContainer *dagger.Container
 	VKContainer        *dagger.Container
+	PluginContainer    *dagger.Container
 }
 
 // New initializes the Dagger module at each call
@@ -67,7 +68,7 @@ func New(name string,
 	// +default="ghcr.io/intertwin-eu/interlink/interlink:0.3.1-rc1"
 	InterlinkRef string,
 	// +optional
-	// +default="ghcr.io/intertwin-eu/interlink-sidecar-slurm/interlink-sidecar-slurm:0.3.2"
+	// +default="ghcr.io/intertwin-eu/interlink-sidecar-slurm/interlink-sidecar-slurm:0.3.6"
 	pluginRef string,
 ) *Interlink {
 
@@ -112,14 +113,14 @@ func (m *Interlink) NewInterlink(
 
 	var err error
 	if pluginEndpoint == nil {
-		plugin := dag.Container().From(m.PluginRef).
+		m.PluginContainer = dag.Container().From(m.PluginRef).
 			WithFile("/etc/interlink/InterLinkConfig.yaml", pluginConfig).
 			WithEnvVariable("SLURMCONFIGPATH", "/etc/interlink/InterLinkConfig.yaml").
 			WithEnvVariable("SHARED_FS", "true").
 			WithExposedPort(4000).
 			WithExec([]string{}, dagger.ContainerWithExecOpts{UseEntrypoint: true, InsecureRootCapabilities: true})
 
-		pluginEndpoint, err = plugin.AsService().Start(ctx)
+		pluginEndpoint, err = m.PluginContainer.AsService().Start(ctx)
 		if err != nil {
 			return nil, err
 		}
@@ -400,7 +401,9 @@ func (m *Interlink) Test(
 		return nil, err
 	}
 
-	result := c.WithExec([]string{"bash", "-c", "source .venv/bin/activate && export KUBECONFIG=/.kube/config  && pytest -vk 'not rclone and not limits'"})
+	// result := c.WithExec([]string{"bash", "-c", "source .venv/bin/activate && export KUBECONFIG=/.kube/config  && pytest -vk 'not rclone and not limits'"})
+	//_ = c.WithExec([]string{"bash", "-c", "source .venv/bin/activate && export KUBECONFIG=/.kube/config  && pytest -vk 'hello'"})
+	result := c.WithExec([]string{"bash", "-c", "source .venv/bin/activate && export KUBECONFIG=/.kube/config  && pytest -vk 'hello'"})
 
 	return result, nil
 

diff --git a/ci/manifests/interlink-config-local.yaml b/ci/manifests/interlink-config-local.yaml
@@ -0,0 +1,17 @@
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: "interlink-config"
+#   namespace: interlink 
+# data:
+#   InterLinkConfig.yaml: |
+    #InterlinkAddress: "unix:///var/run/interlink.socket"
+InterlinkAddress: "http://0.0.0.0"
+InterlinkPort: "3000"
+#SidecarURL: "http://plugin"
+SidecarURL: "http://0.0.0.0"
+SidecarPort: "4000"
+VerboseLogging: true
+ErrorsOnlyLogging: false
+ExportPodData: true
+DataRootFolder: "~/.interlink"
diff --git a/cmd/installer/main.go b/cmd/installer/main.go
@@ -62,6 +62,7 @@ type dataStruct struct {
 	Namespace        string      `yaml:"kubernetes_namespace,omitempty"`
 	VKLimits         Resources   `yaml:"node_limits"`
 	OAUTH            oauthStruct `yaml:"oauth,omitempty"`
+	HTTPInsecure     bool        `default:"true" yaml:"insecure_http"`
 }
 
 func evalManifest(path string, dataStruct dataStruct) (string, error) {
@@ -128,6 +129,7 @@ func root(cmd *cobra.Command, _ []string) error {
 				GitHUBUser:    "myusername",
 				Issuer:        "https://github.com/oauth",
 			},
+			HTTPInsecure: true,
 		}
 
 		yamlData, err := yaml.Marshal(dumpConfig)

diff --git a/cmd/installer/templates/interlink-install.sh b/cmd/installer/templates/interlink-install.sh
@@ -3,32 +3,30 @@
 OS=$(uname -s)
 
 case "$OS" in
-    Darwin)
-        OS=MacOS
-        ;;
+Darwin)
+  OS=MacOS
+  ;;
 esac
 
 OSARCH=$(uname -m)
 case "$OSARCH" in
-    x86_64)
-        OSARCH=amd64
-        ;;
-    aarch64)
-        OSARCH=arm64
-        ;;
+x86_64)
+  OSARCH=amd64
+  ;;
+aarch64)
+  OSARCH=arm64
+  ;;
 esac
 
-
 #echo $OS
 
-OS_LOWER=$(uname -s  | tr '[:upper:]' '[:lower:]')
+OS_LOWER=$(uname -s | tr '[:upper:]' '[:lower:]')
 
-install () {
+install() {
   mkdir -p $HOME/.interlink/logs || exit 1
   mkdir -p $HOME/.interlink/bin || exit 1
   mkdir -p $HOME/.interlink/config || exit 1
 
-
   # TODO download also service files for systemd
 
   cat <<EOF >>$HOME/.interlink/config/InterLinkConfig.yaml
@@ -42,152 +40,152 @@ ExportPodData: true
 DataRootFolder: "~/.interlink"
 EOF
 
-INTERLINK_OS=$(uname -s)
-INTERLINK_ARCH=$(uname -m)
+  INTERLINK_OS=$(uname -s)
+  INTERLINK_ARCH=$(uname -m)
 
-# aarch64 is arm64 in golang. The goreleaser does not consider aarch64 as a different architecture.
-if [ "$INTERLINK_ARCH" = "aarch64" ]; then
-        INTERLINK_ARCH="arm64"
-fi
+  # aarch64 is arm64 in golang. The goreleaser does not consider aarch64 as a different architecture.
+  if [ "$INTERLINK_ARCH" = "aarch64" ]; then
+    INTERLINK_ARCH="arm64"
+  fi
 
   echo "=== Configured to reach sidecar service on unix://${HOME}/.interlink/plugin.sock. You can edit this behavior changing $HOME/.interlink/config/InterLinkConfig.yaml file. ==="
 
   ## Download binaries to $HOME/.local/interlink/
   echo "curl --fail -L -o ${HOME}/.interlink/bin/interlink https://github.com/interTwin-eu/interLink/releases/download/{{.InterLinkVersion}}/interlink_${INTERLINK_OS}_${INTERLINK_ARCH}"
 
   {
-      {
-          curl --fail -L -o ${HOME}/.interlink/bin/interlink https://github.com/interTwin-eu/interLink/releases/download/{{.InterLinkVersion}}/interlink_${INTERLINK_OS}_${INTERLINK_ARCH} 
-          chmod +x ${HOME}/.interlink/bin/interlink
-      } || {
-          echo "Error downloading InterLink binaries, exiting..."
-          exit 1
-      }
-  } 
+    {
+      curl --fail -L -o ${HOME}/.interlink/bin/interlink https://github.com/interTwin-eu/interLink/releases/download/{{.InterLinkVersion}}/interlink_${INTERLINK_OS}_${INTERLINK_ARCH}
+      chmod +x ${HOME}/.interlink/bin/interlink
+    } || {
+      echo "Error downloading InterLink binaries, exiting..."
+      exit 1
+    }
+  }
 
   ## Download oauth2 proxy
   case "$OS" in
   Darwin)
-      go install github.com/oauth2-proxy/oauth2-proxy/v7@latest
-      ;;
+    go install github.com/oauth2-proxy/oauth2-proxy/v7@latest
+    ;;
   Linux)
-      echo "https://github.com/oauth2-proxy/oauth2-proxy/releases/download/v7.6.0/oauth2-proxy-v7.6.0.${OS_LOWER}-$OSARCH.tar.gz"
+    echo "https://github.com/oauth2-proxy/oauth2-proxy/releases/download/v7.6.0/oauth2-proxy-v7.6.0.${OS_LOWER}-$OSARCH.tar.gz"
+    {
       {
-          {
-              curl --fail -L -o ${HOME}/.interlink/bin/oauth2-proxy https://github.com/dciangot/oauth2-proxy/releases/download/v0.0.3/oauth2-proxy_${OS}_$OSARCH
-              chmod +x ${HOME}/.interlink/bin/oauth2-proxy
-          } || {
-              echo "Error downloading OAuth binaries, exiting..."
-              exit 1
-          }
+        curl --fail -L -o ${HOME}/.interlink/bin/oauth2-proxy https://github.com/dciangot/oauth2-proxy/releases/download/v0.0.3/oauth2-proxy_${OS}_$OSARCH
+        chmod +x ${HOME}/.interlink/bin/oauth2-proxy
+      } || {
+        echo "Error downloading OAuth binaries, exiting..."
+        exit 1
       }
+    }
 
-      ;;
+    ;;
   esac
 
   if [[ ! -f ${HOME}/.interlink/config/tls.key || ! -f ${HOME}/.interlink/config/tls.crt ]]; then
 
     openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \
       -keyout ${HOME}/.interlink/config/tls.key \
       -out ${HOME}/.interlink/config/tls.crt \
-      -subj "/CN=interlink.demo"  -addext "subjectAltName=IP:{{.InterLinkIP}}"
+      -subj "/CN=interlink.demo" -addext "subjectAltName=IP:{{.InterLinkIP}}"
 
   fi
 
 }
 
 start() {
-  case "{{.OAUTH.Provider}}" in 
-    oidc)
-      $HOME/.interlink/bin/oauth2-proxy \
-          --client-id "{{.OAUTH.ClientID}}" \
-          --client-secret "\"{{.OAUTH.ClientSecret}}\"" \
-          --oidc-issuer-url "{{.OAUTH.Issuer}}" \
-          --pass-authorization-header true \
-          --provider oidc \
-          --redirect-url http://localhost:8081 \
-          --oidc-extra-audience {{.OAUTH.Audience}} \
-          --upstream unix://${HOME}/.interlink/interlink.sock \
-          --allowed-group {{.OAUTH.Group}} \
-          --validate-url {{.OAUTH.TokenURL}} \
-          --oidc-groups-claim {{.OAUTH.GroupClaim}} \
-          --email-domain=* \
-          --cookie-secret 2ISpxtx19fm7kJlhbgC4qnkuTlkGrshY82L3nfCSKy4= \
-          --skip-auth-route="*='*'" \
-          --force-https \
-          --https-address 0.0.0.0:{{.InterLinkPort}} \
-          --tls-cert-file ${HOME}/.interlink/config/tls.crt \
-          --tls-key-file ${HOME}/.interlink/config/tls.key \
-          --tls-cipher-suite=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 \
-          --skip-jwt-bearer-tokens true > $HOME/.interlink/logs/oauth2-proxy.log 2>&1 &
-
-      echo $! > $HOME/.interlink/oauth2-proxy.pid
-      ;;
-    github)
-      $HOME/.interlink/bin/oauth2-proxy \
-          --client-id {{.OAUTH.ClientID}} \
-          --client-secret {{.OAUTH.ClientSecret}} \
-          --pass-authorization-header true \
-          --provider github \
-          --redirect-url http://localhost:8081 \
-          --upstream unix://$HOME/.interlink/interlink.sock \
-          --email-domain="*" \
-          --github-user="{{.OAUTH.GitHUBUser}}" \
-          --cookie-secret 2ISpxtx19fm7kJlhbgC4qnkuTlkGrshY82L3nfCSKy4= \
-          --skip-auth-route="*='*'" \
-          --force-https \
-          --https-address 0.0.0.0:{{.InterLinkPort}} \
-          --tls-cert-file ${HOME}/.interlink/config/tls.crt \
-          --tls-key-file ${HOME}/.interlink/config/tls.key \
-          --tls-cipher-suite=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 \
-          --skip-jwt-bearer-tokens true > $HOME/.interlink/logs/oauth2-proxy.log 2>&1 &
-
-      echo $! > $HOME/.interlink/oauth2-proxy.pid
-      ;;
+  case "{{.OAUTH.Provider}}" in
+  oidc)
+    $HOME/.interlink/bin/oauth2-proxy \
+      --client-id "{{.OAUTH.ClientID}}" \
+      --client-secret "\"{{.OAUTH.ClientSecret}}\"" \
+      --oidc-issuer-url "{{.OAUTH.Issuer}}" \
+      --pass-authorization-header true \
+      --provider oidc \
+      --redirect-url http://localhost:8081 \
+      --oidc-extra-audience {{.OAUTH.Audience}} \
+      --upstream unix://${HOME}/.interlink/interlink.sock \
+      --allowed-group {{.OAUTH.Group}} \
+      --validate-url {{.OAUTH.TokenURL}} \
+      --oidc-groups-claim {{.OAUTH.GroupClaim}} \
+      --email-domain=* \
+      --cookie-secret 2ISpxtx19fm7kJlhbgC4qnkuTlkGrshY82L3nfCSKy4= \
+      --skip-auth-route="*='*'" \
+      --force-https \
+      --https-address 0.0.0.0:{{.InterLinkPort}} \
+      --tls-cert-file ${HOME}/.interlink/config/tls.crt \
+      --tls-key-file ${HOME}/.interlink/config/tls.key \
+      --tls-cipher-suite=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 \
+      --skip-jwt-bearer-tokens true >$HOME/.interlink/logs/oauth2-proxy.log 2>&1 &
+
+    echo $! >$HOME/.interlink/oauth2-proxy.pid
+    ;;
+  github)
+    $HOME/.interlink/bin/oauth2-proxy \
+      --client-id {{.OAUTH.ClientID}} \
+      --client-secret {{.OAUTH.ClientSecret}} \
+      --pass-authorization-header true \
+      --provider github \
+      --redirect-url http://localhost:8081 \
+      --upstream unix://$HOME/.interlink/interlink.sock \
+      --email-domain="*" \
+      --github-user="{{.OAUTH.GitHUBUser}}" \
+      --cookie-secret 2ISpxtx19fm7kJlhbgC4qnkuTlkGrshY82L3nfCSKy4= \
+      --skip-auth-route="*='*'" \
+      --force-https \
+      --https-address 0.0.0.0:{{.InterLinkPort}} \
+      --tls-cert-file ${HOME}/.interlink/config/tls.crt \
+      --tls-key-file ${HOME}/.interlink/config/tls.key \
+      --tls-cipher-suite=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 \
+      --skip-jwt-bearer-tokens true >$HOME/.interlink/logs/oauth2-proxy.log 2>&1 &
+
+    echo $! >$HOME/.interlink/oauth2-proxy.pid
+    ;;
 
   esac
 
-  ## start interLink 
+  ## start interLink
   export INTERLINKCONFIGPATH=${HOME}/.interlink/config/InterLinkConfig.yaml
-  $HOME/.interlink/bin/interlink &> ${HOME}/.interlink/logs/interlink.log &
-  echo $! > ${HOME}/.interlink/interlink.pid
+  $HOME/.interlink/bin/interlink &>${HOME}/.interlink/logs/interlink.log &
+  echo $! >${HOME}/.interlink/interlink.pid
 
   ## TODO: if RUN_SLURM=1 then manage also slurm
 
 }
 
-stop () {
-    kill $(cat $HOME/.interlink/oauth2-proxy.pid)
-    kill $(cat $HOME/.interlink/interlink.pid)
+stop() {
+  kill $(cat $HOME/.interlink/oauth2-proxy.pid)
+  kill $(cat $HOME/.interlink/interlink.pid)
 }
 
-help () {
-    echo -e "\n\ninstall:      Downloads InterLink and OAuth binaries, as well as InterLink configuration. Files are stored in $HOME/.interlink\n\n"
-    echo -e "start:        Starts the OAuth proxy, the InterLink API.\n"
-    echo -e "stop:         Kills all the previously started processes\n\n"
-    echo -e "restart:      Kills all started processes and start them again\n\n"
-    echo -e "help:         Shows this command list"
+help() {
+  echo -e "\n\ninstall:      Downloads InterLink and OAuth binaries, as well as InterLink configuration. Files are stored in $HOME/.interlink\n\n"
+  echo -e "start:        Starts the OAuth proxy, the InterLink API.\n"
+  echo -e "stop:         Kills all the previously started processes\n\n"
+  echo -e "restart:      Kills all started processes and start them again\n\n"
+  echo -e "help:         Shows this command list"
 }
 
 case "$1" in
-    install)
-        install
-        ;;
-    start) 
-        start
-        ;;
-    stop)
-        stop
-        ;;
-    restart)
-        stop
-        start
-        ;;
-    help)
-        help
-        ;;
-    *)
-        echo -e "You need to specify one of the following commands:"
-        help
-        ;;
+install)
+  install
+  ;;
+start)
+  start
+  ;;
+stop)
+  stop
+  ;;
+restart)
+  stop
+  start
+  ;;
+help)
+  help
+  ;;
+*)
+  echo -e "You need to specify one of the following commands:"
+  help
+  ;;
 esac