Merge branch 'netbirdio:main' into main

.github/workflows/golang-test-linux.yml

@@ -52,6 +52,47 @@ jobs:
       - name: Test
         run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
 
+  benchmark:
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ '386','amd64' ]
+        store: [ 'sqlite', 'postgres' ]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23.x"
+
+
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib libpcap-dev
+
+      - name: Install 32-bit libpcap
+        if: matrix.arch == '386'
+        run: sudo dpkg --add-architecture i386 && sudo apt update && sudo apt-get install -y libpcap0.8-dev:i386
+
+      - name: Install modules
+        run: go mod tidy
+
+      - name: check git status
+        run: git --no-pager diff --exit-code
+
+      - name: Test
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} NETBIRD_STORE_ENGINE=${{ matrix.store }} CI=true go test -run=^$ -bench=. -exec 'sudo --preserve-env=CI,NETBIRD_STORE_ENGINE' -timeout 10m -p 1 ./...
+
   test_client_on_docker:
     runs-on: ubuntu-20.04
     steps:

client/anonymize/anonymize.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 )
 
+const anonTLD = ".domain"
+
 type Anonymizer struct {
 	ipAnonymizer     map[netip.Addr]netip.Addr
 	domainAnonymizer map[string]string
@@ -83,29 +85,39 @@ func (a *Anonymizer) AnonymizeIPString(ip string) string {
 }
 
 func (a *Anonymizer) AnonymizeDomain(domain string) string {
-	if strings.HasSuffix(domain, "netbird.io") ||
-		strings.HasSuffix(domain, "netbird.selfhosted") ||
-		strings.HasSuffix(domain, "netbird.cloud") ||
-		strings.HasSuffix(domain, "netbird.stage") ||
-		strings.HasSuffix(domain, ".domain") {
+	baseDomain := domain
+	hasDot := strings.HasSuffix(domain, ".")
+	if hasDot {
+		baseDomain = domain[:len(domain)-1]
+	}
+
+	if strings.HasSuffix(baseDomain, "netbird.io") ||
+		strings.HasSuffix(baseDomain, "netbird.selfhosted") ||
+		strings.HasSuffix(baseDomain, "netbird.cloud") ||
+		strings.HasSuffix(baseDomain, "netbird.stage") ||
+		strings.HasSuffix(baseDomain, anonTLD) {
 		return domain
 	}
 
-	parts := strings.Split(domain, ".")
+	parts := strings.Split(baseDomain, ".")
 	if len(parts) < 2 {
 		return domain
 	}
 
-	baseDomain := parts[len(parts)-2] + "." + parts[len(parts)-1]
+	baseForLookup := parts[len(parts)-2] + "." + parts[len(parts)-1]
 
-	anonymized, ok := a.domainAnonymizer[baseDomain]
+	anonymized, ok := a.domainAnonymizer[baseForLookup]
 	if !ok {
-		anonymizedBase := "anon-" + generateRandomString(5) + ".domain"
-		a.domainAnonymizer[baseDomain] = anonymizedBase
+		anonymizedBase := "anon-" + generateRandomString(5) + anonTLD
+		a.domainAnonymizer[baseForLookup] = anonymizedBase
 		anonymized = anonymizedBase
 	}
 
-	return strings.Replace(domain, baseDomain, anonymized, 1)
+	result := strings.Replace(baseDomain, baseForLookup, anonymized, 1)
+	if hasDot {
+		result += "."
+	}
+	return result
 }
 
 func (a *Anonymizer) AnonymizeURI(uri string) string {
@@ -152,9 +164,9 @@ func (a *Anonymizer) AnonymizeString(str string) string {
 	return str
 }
 
-// AnonymizeSchemeURI finds and anonymizes URIs with stun, stuns, turn, and turns schemes.
+// AnonymizeSchemeURI finds and anonymizes URIs with ws, wss, rel, rels, stun, stuns, turn, and turns schemes.
 func (a *Anonymizer) AnonymizeSchemeURI(text string) string {
-	re := regexp.MustCompile(`(?i)\b(stuns?:|turns?:|https?://)\S+\b`)
+	re := regexp.MustCompile(`(?i)\b(wss?://|rels?://|stuns?:|turns?:|https?://)\S+\b`)
 
 	return re.ReplaceAllStringFunc(text, a.AnonymizeURI)
 }
@@ -168,10 +180,10 @@ func (a *Anonymizer) AnonymizeDNSLogLine(logEntry string) string {
 		parts := strings.Split(match, `"`)
 		if len(parts) >= 2 {
 			domain := parts[1]
-			if strings.HasSuffix(domain, ".domain") {
+			if strings.HasSuffix(domain, anonTLD) {
 				return match
 			}
-			randomDomain := generateRandomString(10) + ".domain"
+			randomDomain := generateRandomString(10) + anonTLD
 			return strings.Replace(match, domain, randomDomain, 1)
 		}
 		return match

client/anonymize/anonymize_test.go

@@ -67,18 +67,36 @@ func TestAnonymizeDomain(t *testing.T) {
 			`^anon-[a-zA-Z0-9]+\.domain$`,
 			true,
 		},
+		{
+			"Domain with Trailing Dot",
+			"example.com.",
+			`^anon-[a-zA-Z0-9]+\.domain.$`,
+			true,
+		},
 		{
 			"Subdomain",
 			"sub.example.com",
 			`^sub\.anon-[a-zA-Z0-9]+\.domain$`,
 			true,
 		},
+		{
+			"Subdomain with Trailing Dot",
+			"sub.example.com.",
+			`^sub\.anon-[a-zA-Z0-9]+\.domain.$`,
+			true,
+		},
 		{
 			"Protected Domain",
 			"netbird.io",
 			`^netbird\.io$`,
 			false,
 		},
+		{
+			"Protected Domain with Trailing Dot",
+			"netbird.io.",
+			`^netbird\.io.$`,
+			false,
+		},
 	}
 
 	for _, tc := range tests {
@@ -140,8 +158,16 @@ func TestAnonymizeSchemeURI(t *testing.T) {
 		expect string
 	}{
 		{"STUN URI in text", "Connection made via stun:example.com", `Connection made via stun:anon-[a-zA-Z0-9]+\.domain`},
+		{"STUNS URI in message", "Secure connection to stuns:example.com:443", `Secure connection to stuns:anon-[a-zA-Z0-9]+\.domain:443`},
 		{"TURN URI in log", "Failed attempt turn:some.example.com:3478?transport=tcp: retrying", `Failed attempt turn:some.anon-[a-zA-Z0-9]+\.domain:3478\?transport=tcp: retrying`},
+		{"TURNS URI in message", "Secure connection to turns:example.com:5349", `Secure connection to turns:anon-[a-zA-Z0-9]+\.domain:5349`},
+		{"HTTP URI in text", "Visit http://example.com for more", `Visit http://anon-[a-zA-Z0-9]+\.domain for more`},
+		{"HTTPS URI in CAPS", "Visit HTTPS://example.com for more", `Visit https://anon-[a-zA-Z0-9]+\.domain for more`},
 		{"HTTPS URI in message", "Visit https://example.com for more", `Visit https://anon-[a-zA-Z0-9]+\.domain for more`},
+		{"WS URI in log", "Connection established to ws://example.com:8080", `Connection established to ws://anon-[a-zA-Z0-9]+\.domain:8080`},
+		{"WSS URI in message", "Secure connection to wss://example.com", `Secure connection to wss://anon-[a-zA-Z0-9]+\.domain`},
+		{"Rel URI in text", "Relaying to rel://example.com", `Relaying to rel://anon-[a-zA-Z0-9]+\.domain`},
+		{"Rels URI in message", "Relaying to rels://example.com", `Relaying to rels://anon-[a-zA-Z0-9]+\.domain`},
 	}
 
 	for _, tc := range tests {

client/cmd/debug.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -61,6 +62,15 @@ var forCmd = &cobra.Command{
 	RunE:    runForDuration,
 }
 
+var persistenceCmd = &cobra.Command{
+	Use:     "persistence [on|off]",
+	Short:   "Set network map memory persistence",
+	Long:    `Configure whether the latest network map should persist in memory. When enabled, the last known network map will be kept in memory.`,
+	Example: "  netbird debug persistence on",
+	Args:    cobra.ExactArgs(1),
+	RunE:    setNetworkMapPersistence,
+}
+
 func debugBundle(cmd *cobra.Command, _ []string) error {
 	conn, err := getClient(cmd)
 	if err != nil {
@@ -171,6 +181,13 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 
 	time.Sleep(1 * time.Second)
 
+	// Enable network map persistence before bringing the service up
+	if _, err := client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: true,
+	}); err != nil {
+		return fmt.Errorf("failed to enable network map persistence: %v", status.Convert(err).Message())
+	}
+
 	if _, err := client.Up(cmd.Context(), &proto.UpRequest{}); err != nil {
 		return fmt.Errorf("failed to up: %v", status.Convert(err).Message())
 	}
@@ -200,6 +217,13 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to bundle debug: %v", status.Convert(err).Message())
 	}
 
+	// Disable network map persistence after creating the debug bundle
+	if _, err := client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: false,
+	}); err != nil {
+		return fmt.Errorf("failed to disable network map persistence: %v", status.Convert(err).Message())
+	}
+
 	if stateWasDown {
 		if _, err := client.Down(cmd.Context(), &proto.DownRequest{}); err != nil {
 			return fmt.Errorf("failed to down: %v", status.Convert(err).Message())
@@ -219,6 +243,34 @@ func runForDuration(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func setNetworkMapPersistence(cmd *cobra.Command, args []string) error {
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			log.Errorf(errCloseConnection, err)
+		}
+	}()
+
+	persistence := strings.ToLower(args[0])
+	if persistence != "on" && persistence != "off" {
+		return fmt.Errorf("invalid persistence value: %s. Use 'on' or 'off'", args[0])
+	}
+
+	client := proto.NewDaemonServiceClient(conn)
+	_, err = client.SetNetworkMapPersistence(cmd.Context(), &proto.SetNetworkMapPersistenceRequest{
+		Enabled: persistence == "on",
+	})
+	if err != nil {
+		return fmt.Errorf("failed to set network map persistence: %v", status.Convert(err).Message())
+	}
+
+	cmd.Printf("Network map persistence set to: %s\n", persistence)
+	return nil
+}
+
 func getStatusOutput(cmd *cobra.Command) string {
 	var statusOutputString string
 	statusResp, err := getStatus(cmd.Context())

client/cmd/pprof.go

@@ -0,0 +1,33 @@
+//go:build pprof
+// +build pprof
+
+package cmd
+
+import (
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func init() {
+	addr := pprofAddr()
+	go pprof(addr)
+}
+
+func pprofAddr() string {
+	listenAddr := os.Getenv("NB_PPROF_ADDR")
+	if listenAddr == "" {
+		return "localhost:6969"
+	}
+
+	return listenAddr
+}
+
+func pprof(listenAddr string) {
+	log.Infof("listening pprof on: %s\n", listenAddr)
+	if err := http.ListenAndServe(listenAddr, nil); err != nil {
+		log.Fatalf("Failed to start pprof: %v", err)
+	}
+}

client/cmd/root.go

@@ -155,6 +155,7 @@ func init() {
 	debugCmd.AddCommand(logCmd)
 	logCmd.AddCommand(logLevelCmd)
 	debugCmd.AddCommand(forCmd)
+	debugCmd.AddCommand(persistenceCmd)
 
 	upCmd.PersistentFlags().StringSliceVar(&natExternalIPs, externalIPMapFlag, nil,
 		`Sets external IPs maps between local addresses and interfaces.`+