Self-hosted release 0.20.0

orb-community · Oct 18, 2022 · 7a84657 · 7a84657
2 parents 3a226b2 + 9d93fd6
commit 7a84657
Show file tree

Hide file tree

Showing 139 changed files with 7,282 additions and 3,429 deletions.
diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml
@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Get branch name
         shell: bash
@@ -95,17 +95,23 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Replace crashpad params
+      - name: Replace crashpad and geo-db params
         run: |
           ESCAPED_REPLACE_LINE1_NEW=$(printf '%s\n' "pvOptions = append(pvOptions, \"--cp-token\", \"${{ secrets.CRASHPAD_TOKEN }}\")" | sed -e 's/[\/&]/\\&/g')
           ESCAPED_REPLACE_LINE2_NEW=$(printf '%s\n' "pvOptions = append(pvOptions, \"--cp-url\", \"${{ secrets.CRASHPAD_URL }}\")" | sed -e 's/[\/&]/\\&/g')
           ESCAPED_REPLACE_LINE3_NEW=$(printf '%s\n' "pvOptions = append(pvOptions, \"--cp-path\", \"/usr/local/sbin/crashpad_handler\")" | sed -e 's/[\/&]/\\&/g')
+          ESCAPED_REPLACE_LINE4_NEW=$(printf '%s\n' "pvOptions = append(pvOptions, \"--default-geo-city\", \"/geo-db/city.mmdb\")" | sed -e 's/[\/&]/\\&/g')
+          ESCAPED_REPLACE_LINE5_NEW=$(printf '%s\n' "pvOptions = append(pvOptions, \"--default-geo-asn\", \"/geo-db/asn.mmdb\")" | sed -e 's/[\/&]/\\&/g')
           ESCAPED_REPLACE_LINE1_OLD=$(printf '%s\n' "// pvOptions = append(pvOptions, \"--cp-token\", PKTVISOR_CP_TOKEN)" | sed -e 's/[\/&]/\\&/g')
           ESCAPED_REPLACE_LINE2_OLD=$(printf '%s\n' "// pvOptions = append(pvOptions, \"--cp-url\", PKTVISOR_CP_URL)" | sed -e 's/[\/&]/\\&/g')
           ESCAPED_REPLACE_LINE3_OLD=$(printf '%s\n' "// pvOptions = append(pvOptions, \"--cp-path\", PKTVISOR_CP_PATH)" | sed -e 's/[\/&]/\\&/g')
+          ESCAPED_REPLACE_LINE4_OLD=$(printf '%s\n' "// pvOptions = append(pvOptions, \"--default-geo-city\", \"/geo-db/city.mmdb\")" | sed -e 's/[\/&]/\\&/g')
+          ESCAPED_REPLACE_LINE5_OLD=$(printf '%s\n' "// pvOptions = append(pvOptions, \"--default-geo-asn\", \"/geo-db/asn.mmdb\")" | sed -e 's/[\/&]/\\&/g')
           sed -i -e "s/$ESCAPED_REPLACE_LINE1_OLD/$ESCAPED_REPLACE_LINE1_NEW/g" agent/backend/pktvisor/pktvisor.go      
           sed -i -e "s/$ESCAPED_REPLACE_LINE2_OLD/$ESCAPED_REPLACE_LINE2_NEW/g" agent/backend/pktvisor/pktvisor.go 
           sed -i -e "s/$ESCAPED_REPLACE_LINE3_OLD/$ESCAPED_REPLACE_LINE3_NEW/g" agent/backend/pktvisor/pktvisor.go
+          sed -i -e "s/$ESCAPED_REPLACE_LINE4_OLD/$ESCAPED_REPLACE_LINE4_NEW/g" agent/backend/pktvisor/pktvisor.go
+          sed -i -e "s/$ESCAPED_REPLACE_LINE5_OLD/$ESCAPED_REPLACE_LINE5_NEW/g" agent/backend/pktvisor/pktvisor.go
 
       - name: Build orb-agent
         shell: bash

diff --git a/.github/workflows/go-develop.yml b/.github/workflows/go-develop.yml
@@ -18,6 +18,7 @@ jobs:
       orb: ${{ steps.filter.outputs.orb }}
       migrate: ${{ steps.filter.outputs.migrate }}
       ui: ${{ steps.filter.outputs.ui }}
+      docs: ${{ steps.filter.outputs.docs }}
       VERSION: ${{ env.VERSION }}
     steps:
       - uses: actions/checkout@v2
@@ -42,6 +43,10 @@ jobs:
               - 'cmd/sinker/**'
             ui: 
               - 'ui/**'
+            docs:
+              - 'fleet/api/http/openapi.yaml'
+              - 'sinks/api/http/openapi.yaml'
+              - 'policies/api/openapi.yaml'
 
       - name: Set branch name
         shell: bash
@@ -80,6 +85,19 @@ jobs:
         uses: ./.github/actions/go-report
         with:
           go_report_threshold: 90.1 #grade A+
+
+  update-api-docs:
+    needs: prebuild
+    runs-on: ubuntu-latest
+    steps:
+    - name: Repository Dispatch
+      uses: peter-evans/repository-dispatch@v2
+      if: ${{ needs.prebuild.outputs.docs == 'true' && github.event_name != 'pull_request' }}
+      with:
+        token: ${{ secrets.GH_ORB_ACCESS_TOKEN }}
+        repository: ns1labs/orb-website
+        event-type: build-docs
+        client-payload: '{"branch_name": "main"}'
 
   test-agent:
     runs-on: ubuntu-latest
@@ -89,7 +107,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.agent == 'true' }}
@@ -103,7 +121,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         run: |
@@ -119,7 +137,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
@@ -133,7 +151,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
@@ -147,7 +165,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}

diff --git a/.github/workflows/go-main.yml b/.github/workflows/go-main.yml
@@ -65,7 +65,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         run: SERVICE=agent make test_service
@@ -78,7 +78,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
@@ -92,7 +92,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
@@ -106,7 +106,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
@@ -120,7 +120,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}

diff --git a/.github/workflows/go-production.yml b/.github/workflows/go-production.yml
@@ -68,7 +68,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.agent == 'true' }}
@@ -82,7 +82,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         run: |
@@ -98,7 +98,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
@@ -112,7 +112,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
@@ -126,7 +126,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.19
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}

diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,8 @@
 build/*
 localconfig/
 
+**/.env
+
 # /ui - UI dependencies
 ui/node_modules
 ui/dist
@@ -27,4 +29,6 @@ ui/.env
 ui/src/environments/environment.env.ts
 
 python-test/behave_orb
-test_agent_name*
+test_agent_name*
+docker/otel-collector-config.yaml
+!docker/otel-collector-config.yaml.tpl
diff --git a/Makefile b/Makefile
@@ -5,6 +5,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+include docker/.env
 
 # expects to be set as env var
 PRODUCTION_AGENT_REF_TAG ?= latest
@@ -16,7 +17,7 @@ PKTVISOR_DEBUG_TAG ?= latest-develop-debug
 DOCKER_IMAGE_NAME_PREFIX ?= orb
 DOCKERHUB_REPO = ns1labs
 BUILD_DIR = build
-SERVICES = fleet policies sinks sinker migrate
+SERVICES = fleet policies sinks sinker migrate maestro
 DOCKERS = $(addprefix docker_,$(SERVICES))
 DOCKERS_DEV = $(addprefix docker_dev_,$(SERVICES))
 CGO_ENABLED ?= 0
@@ -114,6 +115,12 @@ build_docker:
 run:
 	docker-compose -f docker/docker-compose.yml up -d
 
+run-otel:
+	docker-compose -f docker/docker-compose.yml -f docker/dc-zp-kafka.yml -f docker/dc-sinker-otelcol.yml up -d
+
+stop-otel:
+	docker-compose -f docker/docker-compose.yml -f docker/dc-zp-kafka.yml -f docker/dc-sinker-otelcol.yml down
+
 agent_bin:
 	$(call compile_service_linux,agent)
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.19.0
+0.20.0
diff --git a/agent/agent.go b/agent/agent.go
@@ -7,6 +7,7 @@ package agent
 import (
 	"context"
 	"errors"
+	"fmt"
 	"github.com/eclipse/paho.mqtt.golang"
 	"github.com/fatih/structs"
 	"github.com/google/uuid"
@@ -39,6 +40,7 @@ type orbAgent struct {
 	client            mqtt.Client
 	db                *sqlx.DB
 	backends          map[string]backend.Backend
+	backendState      map[string]*backend.State
 	cancelFunction    context.CancelFunc
 	rpcFromCancelFunc context.CancelFunc
 	// TODO: look for a better way to do this, context shouldn't be inside structs
@@ -100,6 +102,7 @@ func (a *orbAgent) startBackends(agentCtx context.Context) error {
 		return errors.New("no backends specified")
 	}
 	a.backends = make(map[string]backend.Backend, len(a.config.OrbAgent.Backends))
+	a.backendState = make(map[string]*backend.State)
 	for name, configurationEntry := range a.config.OrbAgent.Backends {
 		if !backend.HaveBackend(name) {
 			return errors.New("specified backend does not exist: " + name)
@@ -113,6 +116,10 @@ func (a *orbAgent) startBackends(agentCtx context.Context) error {
 			return err
 		}
 		a.backends[name] = be
+		a.backendState[name] = &backend.State{
+			Status:        backend.Unknown,
+			LastRestartTS: time.Now(),
+		}
 	}
 	return nil
 }
@@ -155,20 +162,20 @@ func (a *orbAgent) Start(ctx context.Context, cancelFunc context.CancelFunc) err
 		return err
 	}
 
-	a.startHearbeat()
+	a.logonWithHearbeat()
 
 	return nil
 }
 
-func (a *orbAgent) startHearbeat() {
+func (a *orbAgent) logonWithHearbeat() {
 	a.hbTicker = time.NewTicker(HeartbeatFreq)
 	a.hbDone = make(chan bool)
 	heartbeatCtx, hbcancelFunc := a.extendContext("heartbeat")
 	go a.sendHeartbeats(heartbeatCtx, hbcancelFunc)
 }
 
-func (a *orbAgent) stopHeartbeat(ctx context.Context) {
-	a.logger.Debug("stopping heartbeat", zap.Any("routine", ctx.Value("routine")))
+func (a *orbAgent) logoffWithHeartbeat(ctx context.Context) {
+	a.logger.Debug("stopping heartbeat, going offline status", zap.Any("routine", ctx.Value("routine")))
 	a.hbTicker.Stop()
 	if a.rpcFromCancelFunc != nil {
 		a.rpcFromCancelFunc()
@@ -188,14 +195,14 @@ func (a *orbAgent) Stop(ctx context.Context) {
 		a.rpcFromCancelFunc()
 	}
 	for name, b := range a.backends {
-		if state, _, _ := b.GetState(); state == backend.Running {
+		if state, _, _ := b.GetRunningStatus(); state == backend.Running {
 			a.logger.Debug("stopping backend", zap.String("backend", name))
 			if err := b.Stop(ctx); err != nil {
 				a.logger.Error("error while stopping the backend", zap.String("backend", name))
 			}
 		}
 	}
-	a.stopHeartbeat(ctx)
+	a.logoffWithHeartbeat(ctx)
 	if a.client != nil && a.client.IsConnected() {
 		a.client.Disconnect(250)
 	}
@@ -209,41 +216,28 @@ func (a *orbAgent) Stop(ctx context.Context) {
 	defer a.cancelFunction()
 }
 
-func (a *orbAgent) sanityCheck(ctx context.Context) error {
-	for name, b := range a.backends {
-		state, _, err := b.GetState()
-		if err != nil {
-			a.logger.Error("error in backend", zap.String("backend", name), zap.String("state", state.String()))
-			err2 := a.RestartBackend(ctx, name, "backend with error")
-			if err2 != nil {
-				a.logger.Error("error restarting backend", zap.String("backend", name), zap.String("state", state.String()))
-				a.Stop(ctx)
-				return errors.New("error restarting backend: " + err2.Error())
-			}
-		}
-	}
-	return nil
-}
-
 func (a *orbAgent) RestartBackend(ctx context.Context, name string, reason string) error {
 	if !backend.HaveBackend(name) {
 		return errors.New("specified backend does not exist: " + name)
 	}
 
 	be := a.backends[name]
 	a.logger.Info("restarting backend", zap.String("backend", name), zap.String("reason", reason))
+	a.backendState[name].RestartCount += 1
+	a.backendState[name].LastRestartTS = time.Now()
+	a.backendState[name].LastRestartReason = reason
 	a.logger.Info("removing policies", zap.String("backend", name))
 	if err := a.policyManager.RemoveBackendPolicies(be, true); err != nil {
 		a.logger.Error("failed to remove policies", zap.String("backend", name), zap.Error(err))
 	}
 	a.logger.Info("resetting backend", zap.String("backend", name))
 	if err := be.FullReset(ctx); err != nil {
+		a.backendState[name].LastError = fmt.Sprintf("failed to reset backend: %v", err)
 		a.logger.Error("failed to reset backend", zap.String("backend", name), zap.Error(err))
 	}
 	err := a.sendAgentPoliciesReq()
 	if err != nil {
 		a.logger.Error("failed to send agent policies request", zap.Error(err))
-		return nil
 	}
 	return nil
 }
@@ -279,12 +273,12 @@ func (a *orbAgent) RestartAll(ctx context.Context, reason string) error {
 		}
 	}
 	a.logger.Info("restarting comms", zap.String("reason", reason))
-	a.stopHeartbeat(ctx)
+	a.logoffWithHeartbeat(ctx)
 	err := a.restartComms(ctx)
 	if err != nil {
 		a.logger.Error("failed to restart comms", zap.Error(err))
 	}
-	a.startHearbeat()
+	a.logonWithHearbeat()
 	a.logger.Info("all backends and comms were restarted")
 
 	return nil