Merge pull request #1490 from openziti/release-next

Release 0.31.0

.github/workflows/fablab-db-creation.yml

@@ -0,0 +1,61 @@
+name: fablab db-creation workflow
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+env:
+  GOFLAGS: "-trimpath"
+  GOX_OUTPUT: "release/{{.Arch}}/{{.OS}}/{{.Dir}}"
+  GOX_TEST_OUTPUT: "test/{{.Arch}}/{{.OS}}/bin/{{.Dir}}"
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: "us-east-1"
+  gh_ci_key: ${{ secrets.GH_CI_KEY }}
+  S3_KEY: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  S3_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+jobs:
+  build:
+    name: Build and Run
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ziti
+        uses: actions/checkout@v3
+        with:
+          path: ziti
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.20.x'
+
+      - name: Install Ziti CI
+        uses: openziti/ziti-ci@v1
+
+      - name: Configure Git
+        run: |
+          cd ziti
+          $(go env GOPATH)/bin/ziti-ci configure-git
+
+      - name: Pull ZITI_VERSION and set as $GITHUB_ENV for use with fablab
+        run: |
+          cd ziti
+          version="$($(go env GOPATH)/bin/ziti-ci -q get-current-version)"
+          echo "Ziti Version: $version"
+          echo "ZITI_VERSION=$version" >> $GITHUB_ENV
+
+      - name: Build and Run
+        run: |
+          cd ziti/zititest/models/db-creation
+          go build -o db-creation main.go
+          echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV"
+          ./db-creation create db-creation
+          ./db-creation up
+
+      - name: Teardown
+        if: always()
+        run: |
+          cd ziti/zititest/models/db-creation          
+          ./db-creation dispose

ADOPTERS.md

@@ -27,3 +27,4 @@ Here's the list of projects using and adopting OpenZiti
 | KubeZT - Zero Trust Kubernetes   | https://KubeZT.com/                           | <img src="https://kubezt-public.s3-us-gov-east-1.amazonaws.com/github-org-logo.png" width="100px"> | KubeZT is an on-demand Kubernetes environment that enables developers to build and deploy highly secure applications for high-compliance organizations. |
 | Analytics HQ                     | https://AnalyticsHQ.com/                      | <img src="https://ahq-public.s3-us-gov-west-1.amazonaws.com/ahq-logo.png" width="100px"> | Analytics HQ is a next-generation unified platform built for modern data management and advanced analytics. |
 | PITS Global Data Recovery Services | https://www.pitsdatarecovery.net/ | <img src="https://www.pitsdatarecovery.net/wp-content/uploads/2020/09/pits-logo.svg" width="100px"> | PITS Global Data Recovery Services is a data recovery company in the United States that offers services for recovering data from hard drives, SSDs, flash drives, RAID arrays and more.  |
+| KEOIC                            |  http://www.keoic.com/                        | <img src="https://static.wixstatic.com/media/300f00_ebca9ecefd8743eb82769e1d3079e55a~mv2.jpg" width="100px"> | KEO International Consultants is a multifaceted AEC firm with a presence across the Middle East and Europe. KEO uses the CloudZiti platform to facilitate zero-trust connections for users, devices, and applications throughout their worldwide network of branch offices and sites. |

CHANGELOG.md

@@ -1,3 +1,61 @@
+# Release 0.31.0
+
+## What's New
+
+* Rate limited for model changes
+
+## Rate Limiter for Model Changes
+
+To prevent the controller from being overwhelmed by a flood of changes, a rate limiter
+can be enabled in the configuration file. A maximum number of queued changes can also
+be configured. The rate limited is disabled by default for now. If not specified the
+default number of queued changes is 100.
+
+When the rate limit is hit, an error will be returned. If the request came in from 
+the REST API, the response will use HTTP status code 429 (too many requests). 
+
+The OpenAPI specs have been updated, so if you're using a generated client to make
+REST calls, it's recommened that you regenerate your client.
+
+
+```
+commandRateLimiter:
+    enabled:   true
+    maxQueued: 100
+```
+
+If the rate limiter is enabled, the following metrics will be produced:
+
+* `command.limiter.queued_count` - guage of the current number of queued operations
+* `command.limiter.work_timer` - timer for operations. Includes the following:
+    * A histogram of how long operations take to complete 
+    * A meter showing that rate at which operations are executed
+    * A count of how many operations have been executed
+
+## Component Updates and Bug Fixes
+
+* github.com/openziti/agent: [v1.0.15 -> v1.0.16](https://github.com/openziti/agent/compare/v1.0.15...v1.0.16)
+* github.com/openziti/channel/v2: [v2.0.101 -> v2.0.105](https://github.com/openziti/channel/compare/v2.0.101...v2.0.105)
+* github.com/openziti/edge-api: [v0.25.38 -> v0.26.0](https://github.com/openziti/edge-api/compare/v0.25.38...v0.26.0)
+    * [Issue #49](https://github.com/openziti/edge-api/issues/49) - Add 429 responses to allow indicating that the server is too busy
+
+* github.com/openziti/identity: [v1.0.64 -> v1.0.66](https://github.com/openziti/identity/compare/v1.0.64...v1.0.66)
+* github.com/openziti/metrics: [v1.2.36 -> v1.2.37](https://github.com/openziti/metrics/compare/v1.2.36...v1.2.37)
+* github.com/openziti/sdk-golang: [v0.20.122 -> v0.20.129](https://github.com/openziti/sdk-golang/compare/v0.20.122...v0.20.129)
+    * [Issue #443](https://github.com/openziti/sdk-golang/issues/443) - Don't send close in reponse to a close on a listener
+
+* github.com/openziti/secretstream: [v0.1.12 -> v0.1.13](https://github.com/openziti/secretstream/compare/v0.1.12...v0.1.13)
+* github.com/openziti/storage: [v0.2.20 -> v0.2.23](https://github.com/openziti/storage/compare/v0.2.20...v0.2.23)
+* github.com/openziti/transport/v2: [v2.0.109 -> v2.0.113](https://github.com/openziti/transport/compare/v2.0.109...v2.0.113)
+* github.com/openziti/ziti: [v0.30.5 -> v0.31.0](https://github.com/openziti/ziti/compare/v0.30.5...v0.31.0)
+    * [Issue #1471](https://github.com/openziti/ziti/issues/1471) - Router links not resilient to controller crash
+    * [Issue #1468](https://github.com/openziti/ziti/issues/1468) - Quickstart quietly fails if password is < 5 characters long
+    * [Issue #1445](https://github.com/openziti/ziti/issues/1445) - Add controller update guardrail
+    * [Issue #1442](https://github.com/openziti/ziti/issues/1442) - Network watchdog not shutting down when controller shuts down
+    * [Issue #1465](https://github.com/openziti/ziti/issues/1465) - Upgrade functions `getZiti` and `performMigration` were only functional on Mac OS, now they are functional for Linux and Mac OSs.
+    * [Issue #1217](https://github.com/openziti/ziti/issues/1217) - Quickstart was improperly handling special characters in `ZITI_PWD`. Special characters are now supported for `ZITI_PWD` in quickstart functions.
+
+
 # Release 0.30.5
 
 ## What's New
@@ -11,6 +69,7 @@ Currently only HTTP Connect proxies which don't require authentication are suppo
 
 **Example using `host.v1`**
 
+```
 {
     "address": "192.168.2.50",
     "port": 1234,
@@ -20,6 +79,7 @@ Currently only HTTP Connect proxies which don't require authentication are suppo
         "type": "http"
     }
 }
+```
 
 
 ## Component Updates and Bug Fixes

common/getziti/github.go

@@ -6,6 +6,7 @@ import (
 	"github.com/blang/semver"
 	"github.com/go-resty/resty/v2"
 	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/foundation/v2/versions"
 	c "github.com/openziti/ziti/ziti/constants"
 	"github.com/pkg/errors"
 	"net/http"
@@ -177,15 +178,21 @@ func DownloadGitHubReleaseAsset(fullUrl string, filepath string) (err error) {
 }
 
 func FindVersionAndInstallGitHubRelease(zitiApp string, zitiAppGitHub string, targetOS, targetArch string, binDir string, version string, verbose bool) error {
-	if version != "" {
+	releaseVersion := version
+	if version != "" && version != "latest" {
 		if _, err := semver.Make(strings.TrimPrefix(version, "v")); err != nil {
 			return err
 		}
 	} else {
 		version = "latest"
+		v, err := GetLatestGitHubReleaseVersion(zitiApp, verbose)
+		if err != nil {
+			return err
+		}
+		releaseVersion = v.String()
 	}
 
-	release, err := GetLatestGitHubReleaseAsset(zitiApp, zitiAppGitHub, version, verbose)
+	release, err := GetLatestGitHubReleaseAsset(zitiApp, zitiAppGitHub, releaseVersion, verbose)
 	if err != nil {
 		return err
 	}
@@ -247,8 +254,20 @@ func InstallGitHubRelease(zitiApp string, release *GitHubReleasesData, binDir st
 		if zitiApp == c.ZITI {
 			count := 0
 			zitiFileName := "ziti-" + version
+			expectedPath := "ziti"
+			if version != "latest" {
+				semVer, err := versions.ParseSemVer(version)
+				if err != nil {
+					return err
+				}
+
+				pathChangedVersion := versions.MustParseSemVer("0.29.0")
+				if semVer.CompareTo(pathChangedVersion) < 0 {
+					expectedPath = "ziti/ziti"
+				}
+			}
 			err = UnTarGz(fullPath, binDir, func(path string) (string, bool) {
-				if path == "ziti/ziti" {
+				if path == expectedPath {
 					count++
 					return zitiFileName, true
 				}

common/getziti/install_ziti.go

@@ -2,25 +2,11 @@ package getziti
 
 import (
 	"fmt"
-	"github.com/blang/semver"
 	c "github.com/openziti/ziti/ziti/constants"
-	"strings"
 )
 
 func InstallZiti(targetVersion, targetOS, targetArch, binDir string, verbose bool) error {
-	var newVersion semver.Version
-
-	if targetVersion != "" {
-		newVersion = semver.MustParse(strings.TrimPrefix(targetVersion, "v"))
-	} else {
-		v, err := GetLatestGitHubReleaseVersion(c.ZITI, verbose)
-		if err != nil {
-			return err
-		}
-		newVersion = v
-	}
-
-	fmt.Println("Attempting to install '" + c.ZITI + "' version: v" + newVersion.String())
+	fmt.Println("Attempting to install '" + c.ZITI + "' version: " + targetVersion)
 	return FindVersionAndInstallGitHubRelease(
-		c.ZITI, c.ZITI, targetOS, targetArch, binDir, "v"+newVersion.String(), verbose)
+		c.ZITI, c.ZITI, targetOS, targetArch, binDir, targetVersion, verbose)
 }

controller/api_impl/helpers.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	openApiErrors "github.com/go-openapi/errors"
 	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/foundation/v2/errorz"
 	"github.com/openziti/ziti/controller/api"
 	apierror2 "github.com/openziti/ziti/controller/apierror"
 	"github.com/openziti/ziti/controller/rest_model"
-	"github.com/openziti/foundation/v2/errorz"
 	"net/http"
 )
 
@@ -124,7 +124,7 @@ func ToRestModel(e *errorz.ApiError, requestId string) *rest_model.APIError {
 			ret.Code = errorz.CouldNotValidateCode
 			ret.Message = errorz.CouldNotValidateMessage
 
-		} else if genericErr, ok := e.Cause.(apierror2.GenericCauseError); ok {
+		} else if genericErr, ok := e.Cause.(*apierror2.GenericCauseError); ok {
 			ret.Cause = &rest_model.APIErrorCause{
 				APIError: rest_model.APIError{
 					Data:    genericErr.DataMap,

controller/apierror/api.go

@@ -25,7 +25,7 @@ type GenericCauseError struct {
 	DataMap map[string]interface{}
 }
 
-func (e GenericCauseError) Error() string {
+func (e *GenericCauseError) Error() string {
 	return e.Message
 }
 

controller/apierror/helpers.go

@@ -375,3 +375,11 @@ func NewEnrollmentExists(enrollmentMethod string) *errorz.ApiError {
 		AppendCause: true,
 	}
 }
+
+func NewTooManyUpdatesError() *errorz.ApiError {
+	return &errorz.ApiError{
+		Code:    ServerTooManyRequestsCode,
+		Message: ServerTooManyRequestsMessage,
+		Status:  ServerTooManyRequestsStatus,
+	}
+}

controller/apierror/messages.go

@@ -194,4 +194,8 @@ const (
 	EnrollmentExistsCode    string = "ENROLLMENT_EXISTS"
 	EnrollmentExistsMessage string = "ENROLLMENT_EXISTS"
 	EnrollmentExistsStatus  int    = http.StatusConflict
+
+	ServerTooManyRequestsCode    string = "SERVER_TOO_MANY_REQUESTS"
+	ServerTooManyRequestsMessage string = "Too many requests to alter state have been issued. Please slow your request rate or try again later."
+	ServerTooManyRequestsStatus  int    = http.StatusTooManyRequests
 )

controller/command/command.go

@@ -19,9 +19,9 @@ package command
 import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/channel/v2"
-	"github.com/openziti/ziti/controller/change"
 	"github.com/openziti/foundation/v2/debugz"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/controller/change"
 	"github.com/sirupsen/logrus"
 	"reflect"
 )
@@ -56,6 +56,7 @@ type Dispatcher interface {
 // LocalDispatcher should be used when running a non-clustered system
 type LocalDispatcher struct {
 	EncodeDecodeCommands bool
+	Limiter              RateLimiter
 }
 
 func (self *LocalDispatcher) IsLeaderOrLeaderless() bool {
@@ -82,7 +83,7 @@ func (self *LocalDispatcher) Dispatch(command Command) error {
 	if changeCtx == nil {
 		changeCtx = change.New().SetSourceType("unattributed").SetChangeAuthorType(change.AuthorTypeUnattributed)
 	}
-	ctx := changeCtx.NewMutateContext()
+
 	if self.EncodeDecodeCommands {
 		bytes, err := command.Encode()
 		if err != nil {
@@ -92,10 +93,13 @@ func (self *LocalDispatcher) Dispatch(command Command) error {
 		if err != nil {
 			return err
 		}
-		return cmd.Apply(ctx)
+		command = cmd
 	}
 
-	return command.Apply(ctx)
+	return self.Limiter.RunRateLimited(func() error {
+		ctx := changeCtx.NewMutateContext()
+		return command.Apply(ctx)
+	})
 }
 
 // Decoder instances know how to decode encoded commands