Merge pull request #1814 from openziti/release-next

Release v0.33.0

.github/dependabot.yml

@@ -1,7 +1,12 @@
 version: 2
 updates:
-- package-ecosystem: gomod
-  directory: "/"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
+  - package-ecosystem: gomod
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 10

.github/workflows/codeql.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/fablab-db-creation.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout ziti
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: ziti
 

.github/workflows/golangci-lint.yml

@@ -21,7 +21,7 @@ jobs:
         with:
           go-version: '1.21.x'
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3

.github/workflows/main.yml

@@ -39,7 +39,7 @@ jobs:
     runs-on: macos-11
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -57,6 +57,7 @@ jobs:
           go install github.com/mitchellh/gox@latest
           $(go env GOPATH)/bin/ziti-ci generate-build-info common/version/info_generated.go version
           $(go env GOPATH)/bin/gox -cgo -os=darwin -arch=amd64 -output=$GOX_OUTPUT ./...
+          $(go env GOPATH)/bin/gox -cgo -os=darwin -arch=arm64 -output=$GOX_OUTPUT ./...
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -72,7 +73,7 @@ jobs:
     runs-on: windows-2019
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -103,7 +104,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -139,7 +140,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -170,7 +171,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -293,7 +294,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -408,7 +409,7 @@ jobs:
       ZITI_VERSION: ${{ steps.get_version.outputs.ZITI_VERSION }}
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/publish-docker-images.yml

@@ -19,7 +19,7 @@ jobs:
       ZITI_TUNNEL_IMAGE:     ${{ vars.ZITI_TUNNEL_IMAGE || 'docker.io/openziti/ziti-tunnel' }}
     steps:
       - name: Checkout Workspace
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download Linux Release Artifacts
         uses: actions/download-artifact@v4

.github/workflows/publish-linux-packages.yml

@@ -46,7 +46,7 @@ jobs:
       ZITI_RPM_PROD_REPO: ${{ vars.ZITI_RPM_PROD_REPO || 'zitipax-openziti-rpm-stable' }}
     steps:
       - name: Checkout Workspace
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download Linux Release Artifacts
         uses: actions/download-artifact@v4

.github/workflows/release-quickstart.yml

@@ -44,7 +44,7 @@ jobs:
           running-workflow-name: release-quickstart
 
       - name: Checkout Workspace
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Ziti CI
         uses: openziti/ziti-ci@v1

.github/workflows/test-cloudfront-proxy.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Shallow checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Python
         shell: bash

.github/workflows/test-quickstart.yml

@@ -19,15 +19,63 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  compose-test:
-    name: Test Compose Quickstart
+  expressInstallTest:
+    name: Test expressInstall() Quickstart
     runs-on: ubuntu-latest
     steps:
       - name: Shallow checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
       - name: Install zsh
         shell: bash
         run: sudo apt-get update && sudo apt-get install --yes zsh
+
       - name: Build and run a quickstart container image
         shell: bash
         run: ./quickstart/test/compose-test.zsh
+
+  allInOneTest:
+    name: Test all-in-one Quickstart
+    runs-on: ubuntu-latest
+    env:
+      ZITI_QUICK_TAG: local  # override default :latest since we're not pulling from registry
+      ARTIFACTS_DIR: ${{ github.workspace }}/build  # output dir for `go build`, input dir for `compose up --build`
+      ZIGGY_UID: 1001  # let container EUID run-as GHA "runner" user to share cache, etc.
+    steps:
+      - name: Shallow checkout
+        uses: actions/checkout@v4
+
+      - name: Install Go
+        id: setup-go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: ./go.mod
+
+      - name: Build ziti executable
+        shell: bash
+        run: |
+          mkdir -pv ${ARTIFACTS_DIR}
+          go build -o ${ARTIFACTS_DIR} ${GITHUB_WORKSPACE}/...
+
+      - name: Run the all-in-one quickstart with locally-built ziti executable
+        shell: bash
+        working-directory: ./quickstart/docker/all-in-one
+        run: docker compose up --build --detach
+
+      - name: Wait for all-in-one quickstart controller then run quickstart tests
+        shell: bash
+        working-directory: ./quickstart/docker/all-in-one
+        env:
+          ZITI_GO_VERSION: ${{ steps.setup-go.outputs.go-version }}
+        run: docker compose --profile test run --rm quickstart-test
+
+      - name: Print debug info
+        if: always()
+        shell: bash
+        working-directory: ./quickstart/docker/all-in-one
+        run: |
+          set +e
+          set -x
+          id runner
+          ls -lAn ${GOCACHE:-${HOME}/.cache/go-build}/ ${GOPATH:-${HOME}/go}/pkg/mod/
+          docker compose --profile test logs

.github/workflows/update-dependency.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

CHANGELOG.md

@@ -1,3 +1,47 @@
+# Release 0.33.0
+
+# What's New
+
+* SDK Terminator stability improvements
+* Minor feature updates and bug fixes
+
+## SDK Terminator stability improvements
+
+This release was focused on creating a chaos test for SDK terminators, running it and fixing any issues found.
+The test repeatedly and randomly restarts the controller, routers and tunnelers then verifies that terminators
+end up in the correct state. 
+
+The following tools were also used/added to aid in diagnosing and fixing issues:
+
+* `ziti fabric validate router-sdk-terminators` 
+    * Compares the controller state with the router state
+* `ziti fabric validate terminators`
+    * Checks each selected terminator to ensure it's still valid on the router and/or sdk
+* `ziti fabric inspect sdk-terminators`
+    * Allows inspecting each routers terminator state
+* `ziti fabric inspect router-messaging`
+    * Allows inspecting what the controller has queued for router state sync and terminator validations
+* `ziti edge validate service-hosting`
+    * Shows how many terminators each identity which can host a service has
+
+Several changes were made to the terminator code to ensure that terminators are properly created and cleaned up.
+The routers now use an adaptive rate limiter to control how fast they send terminator related requests to the
+controller. For this to work properly, the rate limiting on the controller must be enabled, so it can report
+back to the routers when it's got too much work.
+
+## Component Updates and Bug Fixes
+
+* github.com/openziti/edge-api: [v0.26.10 -> v0.26.12](https://github.com/openziti/edge-api/compare/v0.26.10...v0.26.12)
+* github.com/openziti/ziti: [v0.32.2 -> v0.33.0](https://github.com/openziti/ziti/compare/v0.32.2...v0.33.0)
+    * [Issue #1815](https://github.com/openziti/ziti/issues/1815) - Panic if api session sync failed handler is called twice in the router
+    * [Issue #1794](https://github.com/openziti/ziti/issues/1794) - Add SDK terminator chaos test and fix any bugs found as part of chaos testing
+    * [Issue #1781](https://github.com/openziti/ziti/issues/1781) - Improve performance when adding intercepted services
+    * [Issue #1369](https://github.com/openziti/ziti/issues/1369) - Allow filtering by policy type when listing identities for service or services for identity
+    * [Issue #1791](https://github.com/openziti/ziti/issues/1791) - route dial isn't checking for network timeouts correctly
+    * [Issue #1204](https://github.com/openziti/ziti/issues/1204) - ziti cli identity tags related flags misbehaving
+    * [Issue #987](https://github.com/openziti/ziti/issues/987) - "ziti create config router edge" doesn't know about --tunnelerMode proxy
+    * [Issue #652](https://github.com/openziti/ziti/issues/652) - Update CLI script M1 Support when github actions allows
+
 # Release 0.32.2
 
 ## What's New
@@ -26,6 +70,7 @@
     * [Issue #1734](https://github.com/openziti/ziti/issues/1734) - Make API rate limiter enabled by default
     * [Issue #1726](https://github.com/openziti/ziti/issues/1726) - Fix some sdk hosting logging
     * [Issue #1725](https://github.com/openziti/ziti/issues/1725) - Fix panic in entity event processing
+    * [Issue #652](https://github.com/openziti/ziti/issues/652) - CI support for MacOS arm64
 
 # Release 0.32.1
 

common/ctrl_msg/messages.go

@@ -55,6 +55,10 @@ const (
 	CreateCircuitRespCircuitId  = 11
 	CreateCircuitRespAddress    = 12
 	CreateCircuitRespTagsHeader = 13
+
+	HeaderResultErrorCode = 10
+
+	ResultErrorRateLimited = 1
 )
 
 func NewCircuitSuccessMsg(sessionId, address string) *channel.Message {

common/handler_common/common.go

@@ -3,6 +3,7 @@ package handler_common
 import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
+	"github.com/openziti/ziti/common/ctrl_msg"
 	"time"
 )
 
@@ -39,3 +40,20 @@ func SendOpResult(request *channel.Message, ch channel.Channel, op string, messa
 		log.WithError(err).Error("failed to send result")
 	}
 }
+
+func SendServerBusy(request *channel.Message, ch channel.Channel, op string) {
+	log := pfxlog.ContextLogger(ch.Label()).WithField("operation", op)
+	log.Errorf("%v error performing %v: (%s)", ch.LogicalName(), op, "server too busy")
+
+	response := channel.NewResult(false, "server too busy")
+	response.ReplyTo(request)
+	response.Headers.PutUint32Header(ctrl_msg.HeaderResultErrorCode, ctrl_msg.ResultErrorRateLimited)
+	if err := response.WithTimeout(5 * time.Second).SendAndWaitForWire(ch); err != nil {
+		log.WithError(err).Error("failed to send result")
+	}
+}
+
+func WasRateLimited(msg *channel.Message) bool {
+	val, found := msg.GetUint32Header(ctrl_msg.HeaderResultErrorCode)
+	return found && val == ctrl_msg.ResultErrorRateLimited
+}

common/inspect/router_message_inspections.go

@@ -0,0 +1,41 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package inspect
+
+type RouterMessagingState struct {
+	RouterUpdates         []*RouterUpdates         `json:"routerUpdates"`
+	TerminatorValidations []*TerminatorValidations `json:"terminatorValidations"`
+}
+
+type RouterInfo struct {
+	Id   string `json:"id"`
+	Name string `json:"name"`
+}
+
+type RouterUpdates struct {
+	Router         RouterInfo   `json:"router"`
+	Version        uint32       `json:"version"`
+	ChangedRouters []RouterInfo `json:"changedRouters"`
+	SendInProgress bool         `json:"sendInProgress"`
+}
+
+type TerminatorValidations struct {
+	Router          RouterInfo `json:"router"`
+	Terminators     []string   `json:"terminators"`
+	CheckInProgress bool       `json:"checkInProgress"`
+	LastSend        string     `json:"lastSend"`
+}