Merge branch 'main' into fix/remove-ids-from-policy-creation

netbirdio · Dec 23, 2024 · 2d626ea · 2d626ea
2 parents 3a95966 + b48cf1b
commit 2d626ea
Show file tree

Hide file tree

Showing 162 changed files with 14,224 additions and 5,036 deletions.
diff --git a/.github/workflows/golang-test-darwin.yml b/.github/workflows/golang-test-darwin.yml
@@ -21,15 +21,17 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: "1.23.x"
+          cache: false
       - name: Checkout code
         uses: actions/checkout@v4
 
       - name: Cache Go modules
         uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
-          key: macos-go-${{ hashFiles('**/go.sum') }}
+          key: macos-gotest-${{ hashFiles('**/go.sum') }}
           restore-keys: |
+            macos-gotest-
             macos-go-
 
       - name: Install libpcap
@@ -42,4 +44,4 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Test
-        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 ./...
+        run: NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 5m -p 1 $(go list ./... | grep -v /management)
diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml
@@ -11,31 +11,164 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  build-cache:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+          cache: false
+
+      - name: Get Go environment
+        run: |
+          echo "cache=$(go env GOCACHE)" >> $GITHUB_ENV
+          echo "modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV     
+
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: |
+            ${{ env.cache }}
+            ${{ env.modcache }}
+          key: ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }}
+            
+
+      - name: Install dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
+
+      - name: Install 32-bit libpcap
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
+
+      - name: Build client
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: client
+        run: CGO_ENABLED=1 go build .
+
+      - name: Build client 386
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: client
+        run: CGO_ENABLED=1 GOARCH=386 go build -o client-386 .
+
+      - name: Build management
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: management
+        run: CGO_ENABLED=1 go build .
+
+      - name: Build management 386
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: management
+        run: CGO_ENABLED=1 GOARCH=386 go build -o management-386 .
+
+      - name: Build signal
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: signal
+        run: CGO_ENABLED=1 go build .
+
+      - name: Build signal 386
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: signal
+        run: CGO_ENABLED=1 GOARCH=386 go build -o signal-386 .
+
+      - name: Build relay
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: relay
+        run: CGO_ENABLED=1 go build .
+
+      - name: Build relay 386
+        if: steps.cache.outputs.cache-hit != 'true'
+        working-directory: relay
+        run: CGO_ENABLED=1 GOARCH=386 go build -o relay-386 .
+
   test:
+    needs: [build-cache]
     strategy:
       fail-fast: false
       matrix:
         arch: [ '386','amd64' ]
-        store: [ 'sqlite', 'postgres']
     runs-on: ubuntu-22.04
     steps:
       - name: Install Go
         uses: actions/setup-go@v5
         with:
           go-version: "1.23.x"
+          cache: false
 
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Go environment
+        run: |
+          echo "cache=$(go env GOCACHE)" >> $GITHUB_ENV
+          echo "modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV     
 
       - name: Cache Go modules
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          path: |
+            ${{ env.cache }}
+            ${{ env.modcache }}
+          key: ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go-
+            ${{ runner.os }}-gotest-cache-
+
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
+
+      - name: Install 32-bit libpcap
+        if: matrix.arch == '386'
+        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
+
+      - name: Install modules
+        run: go mod tidy
+
+      - name: check git status
+        run: git --no-pager diff --exit-code
+
+      - name: Test
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} CI=true go test -exec 'sudo' -timeout 10m -p 1 $(go list ./... | grep -v /management)
+
+  test_management:
+    needs: [ build-cache ]
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ '386','amd64' ]
+        store: [ 'sqlite', 'postgres']
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+          cache: false
 
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Get Go environment
+        run: |
+          echo "cache=$(go env GOCACHE)" >> $GITHUB_ENV
+          echo "modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV     
+
+      - name: Cache Go modules
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ env.cache }}
+            ${{ env.modcache }}
+          key: ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-gotest-cache-
+
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
 
@@ -50,9 +183,10 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Test
-        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m $(go list ./... | grep /management)
 
   benchmark:
+    needs: [ build-cache ]
     strategy:
       fail-fast: false
       matrix:
@@ -64,18 +198,25 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: "1.23.x"
+          cache: false
 
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Go environment
+        run: |
+          echo "cache=$(go env GOCACHE)" >> $GITHUB_ENV
+          echo "modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV     
 
       - name: Cache Go modules
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          path: |
+            ${{ env.cache }}
+            ${{ env.modcache }}
+          key: ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go-
-
-      - name: Checkout code
-        uses: actions/checkout@v4
+            ${{ runner.os }}-gotest-cache-
 
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
@@ -94,23 +235,32 @@ jobs:
         run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -run=^$ -bench=. -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
 
   test_client_on_docker:
+    needs: [ build-cache ]
     runs-on: ubuntu-20.04
     steps:
       - name: Install Go
         uses: actions/setup-go@v5
         with:
           go-version: "1.23.x"
+          cache: false
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Go environment
+        run: |
+          echo "cache=$(go env GOCACHE)" >> $GITHUB_ENV
+          echo "modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV     
 
       - name: Cache Go modules
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          path: |
+            ${{ env.cache }}
+            ${{ env.modcache }}
+          key: ${{ runner.os }}-gotest-cache-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go-
-
-      - name: Checkout code
-        uses: actions/checkout@v4
+            ${{ runner.os }}-gotest-cache-
 
       - name: Install dependencies
         run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev

diff --git a/.github/workflows/golang-test-windows.yml b/.github/workflows/golang-test-windows.yml
@@ -24,6 +24,23 @@ jobs:
         id: go
         with:
           go-version: "1.23.x"
+          cache: false
+
+      - name: Get Go environment
+        run: |
+          echo "cache=$(go env GOCACHE)" >> $env:GITHUB_ENV
+          echo "modcache=$(go env GOMODCACHE)" >> $env:GITHUB_ENV
+
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ env.cache }}
+            ${{ env.modcache }}
+          key: ${{ runner.os }}-gotest-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-gotest-
+            ${{ runner.os }}-go-
 
       - name: Download wintun
         uses: carlosperate/download-file-action@v2
@@ -42,11 +59,13 @@ jobs:
       - run: choco install -y sysinternals --ignore-checksums
       - run: choco install -y mingw
 
-      - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOMODCACHE=C:\Users\runneradmin\go\pkg\mod
-      - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOCACHE=C:\Users\runneradmin\AppData\Local\go-build
+      - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOMODCACHE=${{ env.cache }}
+      - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe env -w GOCACHE=${{ env.modcache }}
+      - run: PsExec64 -s -w ${{ github.workspace }} C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe mod tidy
+      - run: echo "files=$(go list ./... | ForEach-Object { $_ } | Where-Object { $_ -notmatch '/management' })" >> $env:GITHUB_ENV
 
       - name: test
-        run: PsExec64 -s -w ${{ github.workspace }} cmd.exe /c "C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe test -timeout 10m -p 1 ./... > test-out.txt 2>&1"
+        run: PsExec64 -s -w ${{ github.workspace }} cmd.exe /c "C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe test -timeout 10m -p 1 ${{ env.files }} > test-out.txt 2>&1"
       - name: test output
         if: ${{ always() }}
         run: Get-Content test-out.txt
diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml
@@ -46,7 +46,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev libpcap-dev
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
         with:
           version: latest
-          args: --timeout=12m
+          args: --timeout=12m --out-format colored-line-number
diff --git a/client/Dockerfile b/client/Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:3.20
+FROM alpine:3.21.0
 RUN apk add --no-cache ca-certificates iptables ip6tables
 ENV NB_FOREGROUND_MODE=true
 ENTRYPOINT [ "/usr/local/bin/netbird","up"]

diff --git a/client/anonymize/anonymize.go b/client/anonymize/anonymize.go
@@ -21,6 +21,8 @@ type Anonymizer struct {
 	currentAnonIPv6  netip.Addr
 	startAnonIPv4    netip.Addr
 	startAnonIPv6    netip.Addr
+
+	domainKeyRegex *regexp.Regexp
 }
 
 func DefaultAddresses() (netip.Addr, netip.Addr) {
@@ -36,6 +38,8 @@ func NewAnonymizer(startIPv4, startIPv6 netip.Addr) *Anonymizer {
 		currentAnonIPv6:  startIPv6,
 		startAnonIPv4:    startIPv4,
 		startAnonIPv6:    startIPv6,
+
+		domainKeyRegex: regexp.MustCompile(`\bdomain=([^\s,:"]+)`),
 	}
 }
 
@@ -171,20 +175,15 @@ func (a *Anonymizer) AnonymizeSchemeURI(text string) string {
 	return re.ReplaceAllStringFunc(text, a.AnonymizeURI)
 }
 
-// AnonymizeDNSLogLine anonymizes domain names in DNS log entries by replacing them with a random string.
 func (a *Anonymizer) AnonymizeDNSLogLine(logEntry string) string {
-	domainPattern := `dns\.Question{Name:"([^"]+)",`
-	domainRegex := regexp.MustCompile(domainPattern)
-
-	return domainRegex.ReplaceAllStringFunc(logEntry, func(match string) string {
-		parts := strings.Split(match, `"`)
+	return a.domainKeyRegex.ReplaceAllStringFunc(logEntry, func(match string) string {
+		parts := strings.SplitN(match, "=", 2)
 		if len(parts) >= 2 {
 			domain := parts[1]
 			if strings.HasSuffix(domain, anonTLD) {
 				return match
 			}
-			randomDomain := generateRandomString(10) + anonTLD
-			return strings.Replace(match, domain, randomDomain, 1)
+			return "domain=" + a.AnonymizeDomain(domain)
 		}
 		return match
 	})