Preparation for Otterize Cloud - Merge branch 'develop'

.github/workflows/build.yaml

@@ -10,6 +10,7 @@ on:
   push:
     branches:
       - main
+      - develop
 
 env:
   REGISTRY: 353146681200.dkr.ecr.us-east-1.amazonaws.com/otterize
@@ -31,6 +32,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           submodules: recursive
+          token: ${{ secrets.OTTERIZEBOT_GITHUB_TOKEN }} # required for checking out submodules
 
       - name: Set up Docker Buildx
         id: buildx

.github/workflows/golangci-lint.yml

@@ -0,0 +1,47 @@
+name: golangci-lint
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - main
+  pull_request:
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.19.1'
+      - uses: actions/checkout@v3
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install libpcap-dev # required for the linter to be able to lint github.com/google/gopacket
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          version: v1.50.1
+
+          # Optional: working directory, useful for monorepos
+          working-directory: src
+
+          # Optional: golangci-lint command line arguments.
+          args: --timeout 5m
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true then the all caching functionality will be complete disabled,
+          #           takes precedence over all other caching options.
+#          skip-cache: true
+
+          # Optional: if set to true then the action don't cache or restore ~/go/pkg.
+#          skip-pkg-cache: true
+
+          # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
+#          skip-build-cache: true

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/cloudgraphql"]
+	path = src/cloudgraphql
+	url = https://github.com/otterize/graphql

src/mapper.Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 golang:1.18-alpine as buildenv
+FROM --platform=linux/amd64 golang:1.19-alpine as buildenv
 RUN apk add --no-cache ca-certificates git protoc
 RUN apk add build-base libpcap-dev
 WORKDIR /src

src/mapper/cmd/main.go

@@ -5,6 +5,8 @@ import (
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	"github.com/otterize/intents-operator/src/shared/serviceidresolver"
+	"github.com/otterize/network-mapper/src/mapper/pkg/cloudclient"
+	"github.com/otterize/network-mapper/src/mapper/pkg/clouduploader"
 	"github.com/otterize/network-mapper/src/mapper/pkg/config"
 	"github.com/otterize/network-mapper/src/mapper/pkg/kubefinder"
 	"github.com/otterize/network-mapper/src/mapper/pkg/resolvers"
@@ -13,9 +15,11 @@ import (
 	"github.com/spf13/viper"
 	"net/http"
 	"os"
+	"os/signal"
 	clientconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
+	"syscall"
 	"time"
 )
 
@@ -79,11 +83,22 @@ func main() {
 		logrus.Error(intentHolderCfg)
 		os.Exit(1)
 	}
+
+	cloudConfig := clouduploader.ConfigFromViper()
 	intentsHolder := resolvers.NewIntentsHolder(mgr.GetClient(), intentHolderCfg)
+	cloudClient := clouduploader.NewCloudUploader(intentsHolder, cloudConfig, cloudclient.NewClient)
 	resolver := resolvers.NewResolver(kubeFinder, serviceidresolver.NewResolver(mgr.GetClient()), intentsHolder)
 	_ = resolver.LoadStore(initCtx) // loads the store from the previous run
 	resolver.Register(e)
 
+	if cloudConfig.IsCloudUploadEnabled() {
+		go func() {
+			cloudClientCtx, cloudClientCancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+			defer cloudClientCancel()
+			cloudClient.PeriodicIntentsUpload(cloudClientCtx)
+		}()
+	}
+
 	logrus.Info("Starting api server")
 	err = e.Start("0.0.0.0:9090")
 	if err != nil {

src/mapper/gqlgen.yml

@@ -1,6 +1,6 @@
 # Where are all the schema files located? globs are supported eg  src/**/*.graphqls
 schema:
-  - '../graphql/*'
+  - '../mappergraphql/*.graphql'
 
 # Where should the generated server code go?
 exec:

src/mapper/pkg/cloudclient/cloud_client.go

@@ -0,0 +1,52 @@
+package cloudclient
+
+import (
+	"context"
+	"fmt"
+	"github.com/Khan/genqlient/graphql"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/oauth2"
+)
+
+type FactoryFunction func(ctx context.Context, apiAddress string, tokenSource oauth2.TokenSource) CloudClient
+
+type CloudClient interface {
+	ReportDiscoveredIntents(intents []*DiscoveredIntentInput) bool
+	ReportComponentStatus(component ComponentType)
+}
+
+type CloudClientImpl struct {
+	ctx    context.Context
+	client graphql.Client
+}
+
+func NewClient(ctx context.Context, apiAddress string, tokenSource oauth2.TokenSource) CloudClient {
+	url := fmt.Sprintf("%s/graphql/v1", apiAddress)
+	client := graphql.NewClient(url, oauth2.NewClient(ctx, tokenSource))
+
+	return &CloudClientImpl{
+		client: client,
+		ctx:    ctx,
+	}
+}
+
+func (c *CloudClientImpl) ReportDiscoveredIntents(intents []*DiscoveredIntentInput) bool {
+	logrus.Info("Uploading intents to cloud, count: ", len(intents))
+
+	_, err := ReportDiscoveredIntents(c.ctx, c.client, intents)
+	if err != nil {
+		logrus.Error("Failed to upload intents to cloud ", err)
+		return false
+	}
+
+	return true
+}
+
+func (c *CloudClientImpl) ReportComponentStatus(component ComponentType) {
+	logrus.Info("Uploading component to cloud")
+
+	_, err := ReportComponentStatus(c.ctx, c.client, component)
+	if err != nil {
+		logrus.Error("Failed to upload component to cloud ", err)
+	}
+}

src/mapper/pkg/cloudclient/generate.go

@@ -0,0 +1,4 @@
+package cloudclient
+
+//go:generate go run github.com/Khan/genqlient ./genqlient.yaml
+//go:generate go run github.com/golang/mock/[email protected] -destination=./mocks/mocks.go  -package=cloudclientmocks -source=./cloud_client.go CloudClient