Merge pull request #1068 from percona/release-2.8.0

Release 2.8.0

.github/workflows/ci.yml

@@ -28,11 +28,11 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 120
+    timeout-minutes: 180
     strategy:
       fail-fast: false
       matrix:
-        psmdb: ["5.0", "6.0", "7.0"]
+        psmdb: ["6.0", "7.0", "8.0"]
         test: [logical, physical, incremental, external]
     env:
       PBM_BRANCH: ${{ github.event.inputs.pbm_branch || 'main' }}
@@ -49,7 +49,7 @@ jobs:
 
       - name: Setup environment with PSMDB ${{ matrix.psmdb }} for PBM PR/branch ${{ github.event.pull_request.title || env.PR_NUMBER || env.PBM_BRANCH }}
         run: |
-          PSMDB=percona/percona-server-mongodb:${{ matrix.psmdb }} docker compose build
+          PSMDB=perconalab/percona-server-mongodb:${{ matrix.psmdb }} docker compose build
           docker compose up -d
         working-directory: psmdb-testing/pbm-functional/pytest
 

.github/workflows/codecov.yml

@@ -0,0 +1,25 @@
+name: 'codecov'
+
+on:
+  push:
+    branches:
+      - dev
+
+jobs:
+  go-test:
+    name: runner / go-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.22"
+      - name: test
+        run: go test -v ./... -covermode=atomic -coverprofile=cover.out
+
+      - name: upload coverage report
+        uses: codecov/codecov-action@v4
+        with:
+          file: cover.out
+          fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/reviewdog.yml

@@ -10,7 +10,14 @@ jobs:
         with:
           go-version: "1.22"
       - name: test
-        run: go test -v ./...
+        run: go test -v ./... -covermode=atomic -coverprofile=cover.out
+
+      - name: upload coverage report
+        uses: codecov/codecov-action@v4
+        with:
+          file: cover.out
+          fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   shellcheck:
     name: runner / shellcheck

CODEOWNERS

@@ -1 +1 @@
-* @defbin @boris-ilijic
+* @boris-ilijic @veceraj

Makefile

@@ -112,3 +112,20 @@ install-stest-static:
 	$(ENVS_STATIC) go install -ldflags="$(LDFLAGS_STATIC)" $(BUILD_FLAGS) ./cmd/pbm-speed-test
 install-static-entrypoint:
 	$(ENVS_STATIC) go install -ldflags="$(LDFLAGS_STATIC)" $(BUILD_FLAGS) ./cmd/pbm-agent-entrypoint
+
+# BUILD WITH COVERAGE PROFILING
+build-cover: build-pbm-cover build-agent-cover build-stest-cover
+build-pbm-cover:
+	$(ENVS) go build -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) -o ./bin/pbm ./cmd/pbm
+build-agent-cover:
+	$(ENVS) go build -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) -o ./bin/pbm-agent ./cmd/pbm-agent
+build-stest-cover:
+	$(ENVS) go build -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) -o ./bin/pbm-speed-test ./cmd/pbm-speed-test
+
+install-cover: install-pbm-cover install-agent-cover install-stest-cover
+install-pbm-cover:
+	$(ENVS) go install -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) ./cmd/pbm
+install-agent-cover:
+	$(ENVS) go install -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) ./cmd/pbm-agent
+install-stest-cover:
+	$(ENVS) go install -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) ./cmd/pbm-speed-test

README.md

@@ -1,5 +1,5 @@
 # Percona Backup for MongoDB
-[![Go Report Card](https://goreportcard.com/badge/github.com/percona/percona-backup-mongodb)](https://goreportcard.com/report/github.com/percona/percona-backup-mongodb) [![codecov](https://codecov.io/gh/percona/percona-backup-mongodb/branch/master/graph/badge.svg?token=TiuOmTfp2p)](https://codecov.io/gh/percona/percona-backup-mongodb) [![CLA assistant](https://cla-assistant.percona.com/readme/badge/percona/percona-backup-mongodb)](https://cla-assistant.percona.com/percona/percona-backup-mongodb)
+[![Go Report Card](https://goreportcard.com/badge/github.com/percona/percona-backup-mongodb)](https://goreportcard.com/report/github.com/percona/percona-backup-mongodb) [![CLA assistant](https://cla-assistant.percona.com/readme/badge/percona/percona-backup-mongodb)](https://cla-assistant.percona.com/percona/percona-backup-mongodb)
 
 ![PBM logo](backup-mongo.jpeg)
 

cmd/pbm-agent/agent.go

@@ -456,7 +456,7 @@ func (a *Agent) storStatus(
 		return topo.SubsysStatus{OK: true}
 	}
 
-	stg, err := util.GetStorage(ctx, a.leadConn, log)
+	stg, err := util.GetStorage(ctx, a.leadConn, a.brief.Me, log)
 	if err != nil {
 		return topo.SubsysStatus{Err: fmt.Sprintf("unable to get storage: %v", err)}
 	}

cmd/pbm-agent/delete.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/percona/percona-backup-mongodb/pbm/backup"
 	"github.com/percona/percona-backup-mongodb/pbm/config"
-	"github.com/percona/percona-backup-mongodb/pbm/connect"
 	"github.com/percona/percona-backup-mongodb/pbm/ctrl"
 	"github.com/percona/percona-backup-mongodb/pbm/errors"
 	"github.com/percona/percona-backup-mongodb/pbm/lock"
@@ -97,7 +96,7 @@ func (a *Agent) Delete(ctx context.Context, d *ctrl.DeleteBackupCmd, opid ctrl.O
 		}
 
 		l.Info("deleting backups older than %v", t)
-		err = backup.DeleteBackupBefore(ctx, a.leadConn, t, bcpType)
+		err = backup.DeleteBackupBefore(ctx, a.leadConn, t, bcpType, nodeInfo.Me)
 		if err != nil {
 			l.Error("deleting: %v", err)
 			return
@@ -107,7 +106,7 @@ func (a *Agent) Delete(ctx context.Context, d *ctrl.DeleteBackupCmd, opid ctrl.O
 		ctx := log.SetLogEventToContext(ctx, l)
 
 		l.Info("deleting backup")
-		err := backup.DeleteBackup(ctx, a.leadConn, d.Backup)
+		err := backup.DeleteBackup(ctx, a.leadConn, d.Backup, nodeInfo.Me)
 		if err != nil {
 			l.Error("deleting: %v", err)
 			return
@@ -188,7 +187,7 @@ func (a *Agent) DeletePITR(ctx context.Context, d *ctrl.DeletePITRCmd, opid ctrl
 
 	ts := primitive.Timestamp{T: uint32(t.Unix())}
 	l.Info("deleting pitr chunks older than %v", t)
-	err = deletePITRImpl(ctx, a.leadConn, ts)
+	err = a.deletePITRImpl(ctx, ts)
 	if err != nil {
 		l.Error("deleting: %v", err)
 		return
@@ -260,7 +259,7 @@ func (a *Agent) Cleanup(ctx context.Context, d *ctrl.CleanupCmd, opid ctrl.OPID,
 		l.Error("get config: %v", err)
 	}
 
-	stg, err := util.StorageFromConfig(&cfg.Storage, l)
+	stg, err := util.StorageFromConfig(&cfg.Storage, a.brief.Me, l)
 	if err != nil {
 		l.Error("get storage: " + err.Error())
 	}
@@ -298,16 +297,16 @@ func (a *Agent) Cleanup(ctx context.Context, d *ctrl.CleanupCmd, opid ctrl.OPID,
 		l.Error(err.Error())
 	}
 
-	err = resync.Resync(ctx, a.leadConn, &cfg.Storage)
+	err = resync.Resync(ctx, a.leadConn, &cfg.Storage, a.brief.Me)
 	if err != nil {
 		l.Error("storage resync: " + err.Error())
 	}
 }
 
-func deletePITRImpl(ctx context.Context, conn connect.Client, ts primitive.Timestamp) error {
+func (a *Agent) deletePITRImpl(ctx context.Context, ts primitive.Timestamp) error {
 	l := log.LogEventFromContext(ctx)
 
-	r, err := backup.MakeCleanupInfo(ctx, conn, ts)
+	r, err := backup.MakeCleanupInfo(ctx, a.leadConn, ts)
 	if err != nil {
 		return errors.Wrap(err, "get pitr chunks")
 	}
@@ -316,15 +315,15 @@ func deletePITRImpl(ctx context.Context, conn connect.Client, ts primitive.Times
 		return nil
 	}
 
-	stg, err := util.GetStorage(ctx, conn, l)
+	stg, err := util.GetStorage(ctx, a.leadConn, a.brief.Me, l)
 	if err != nil {
 		return errors.Wrap(err, "get storage")
 	}
 
-	return deleteChunks(ctx, conn, stg, r.Chunks)
+	return a.deleteChunks(ctx, stg, r.Chunks)
 }
 
-func deleteChunks(ctx context.Context, m connect.Client, stg storage.Storage, chunks []oplog.OplogChunk) error {
+func (a *Agent) deleteChunks(ctx context.Context, stg storage.Storage, chunks []oplog.OplogChunk) error {
 	l := log.LogEventFromContext(ctx)
 
 	for _, chnk := range chunks {
@@ -333,7 +332,7 @@ func deleteChunks(ctx context.Context, m connect.Client, stg storage.Storage, ch
 			return errors.Wrapf(err, "delete pitr chunk '%s' (%v) from storage", chnk.FName, chnk)
 		}
 
-		_, err = m.PITRChunksCollection().DeleteOne(
+		_, err = a.leadConn.PITRChunksCollection().DeleteOne(
 			ctx,
 			bson.D{
 				{"rs", chnk.RS},

cmd/pbm-agent/main.go

@@ -49,6 +49,20 @@ func main() {
 		versionFormat = versionCmd.Flag("format", "Output format <json or \"\">").
 				Default("").
 				String()
+
+		logPath = pbmCmd.Flag("log-path", "Path to file").
+			Envar("LOG_PATH").
+			Default("/dev/stderr").
+			String()
+		logJSON = pbmCmd.Flag("log-json", "Enable JSON output").
+			Envar("LOG_JSON").
+			Bool()
+		logLevel = pbmCmd.Flag(
+			"log-level",
+			"Minimal log level based on severity level: D, I, W, E or F, low to high. Choosing one includes higher levels too.").
+			Envar("LOG_LEVEL").
+			Default(log.D).
+			Enum(log.D, log.I, log.W, log.E, log.F)
 	)
 
 	cmd, err := pbmCmd.DefaultEnvars().Parse(os.Args[1:])
@@ -74,19 +88,26 @@ func main() {
 
 	hidecreds()
 
-	fmt.Print(perconaSquadNotice)
+	logOpts := &log.Opts{
+		LogPath:  *logPath,
+		LogLevel: *logLevel,
+		LogJSON:  *logJSON,
+	}
+	l := log.NewWithOpts(nil, "", "", logOpts).NewDefaultEvent()
 
-	err = runAgent(url, *dumpConns)
-	stdlog.Println("Exit:", err)
+	err = runAgent(url, *dumpConns, logOpts)
 	if err != nil {
+		l.Error("Exit: %v", err)
 		os.Exit(1)
 	}
+	l.Info("Exit: <nil>")
 }
 
-func runAgent(mongoURI string, dumpConns int) error {
-	mtLog.SetDateFormat(log.LogTimeFormat)
-	mtLog.SetVerbosity(&options.Verbosity{VLevel: mtLog.DebugLow})
-
+func runAgent(
+	mongoURI string,
+	dumpConns int,
+	logOpts *log.Opts,
+) error {
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill)
 	defer cancel()
 
@@ -95,16 +116,33 @@ func runAgent(mongoURI string, dumpConns int) error {
 		return errors.Wrap(err, "connect to PBM")
 	}
 
+	err = setupNewDB(ctx, leadConn)
+	if err != nil {
+		return errors.Wrap(err, "setup pbm collections")
+	}
+
 	agent, err := newAgent(ctx, leadConn, mongoURI, dumpConns)
 	if err != nil {
 		return errors.Wrap(err, "connect to the node")
 	}
 
-	logger := log.New(agent.leadConn.LogCollection(), agent.brief.SetName, agent.brief.Me)
+	logger := log.NewWithOpts(
+		agent.leadConn,
+		agent.brief.SetName,
+		agent.brief.Me,
+		logOpts)
 	defer logger.Close()
 
 	ctx = log.SetLoggerToContext(ctx, logger)
 
+	mtLog.SetDateFormat(log.LogTimeFormat)
+	mtLog.SetVerbosity(&options.Verbosity{VLevel: mtLog.DebugLow})
+	mtLog.SetWriter(logger)
+
+	logger.Printf(perconaSquadNotice)
+	logger.Printf("log options: log-path=%s, log-level:%s, log-json:%t",
+		logOpts.LogPath, logOpts.LogLevel, logOpts.LogJSON)
+
 	canRunSlicer := true
 	if err := agent.CanStart(ctx); err != nil {
 		if errors.Is(err, ErrArbiterNode) || errors.Is(err, ErrDelayedNode) {
@@ -114,11 +152,6 @@ func runAgent(mongoURI string, dumpConns int) error {
 		}
 	}
 
-	err = setupNewDB(ctx, agent.leadConn)
-	if err != nil {
-		return errors.Wrap(err, "setup pbm collections")
-	}
-
 	agent.showIncompatibilityWarning(ctx)
 
 	if canRunSlicer {

cmd/pbm-agent/oplog.go

@@ -77,7 +77,7 @@ func (a *Agent) OplogReplay(ctx context.Context, r *ctrl.ReplayCmd, opID ctrl.OP
 	}
 
 	l.Info("oplog replay started")
-	rr := restore.New(a.leadConn, a.nodeConn, a.brief, cfg, r.RSMap, 0)
+	rr := restore.New(a.leadConn, a.nodeConn, a.brief, cfg, r.RSMap, 0, 1)
 	err = rr.ReplayOplog(ctx, r, opID, l)
 	if err != nil {
 		if errors.Is(err, restore.ErrNoDataForShard) {

cmd/pbm-agent/pitr.go

@@ -284,7 +284,7 @@ func (a *Agent) pitr(ctx context.Context) error {
 		}
 	}()
 
-	stg, err := util.StorageFromConfig(&cfg.Storage, l)
+	stg, err := util.StorageFromConfig(&cfg.Storage, a.brief.Me, l)
 	if err != nil {
 		if err := lck.Release(); err != nil {
 			l.Error("release lock: %v", err)

cmd/pbm-agent/profile.go

@@ -86,7 +86,7 @@ func (a *Agent) handleAddConfigProfile(
 		return
 	}
 
-	stg, err := util.StorageFromConfig(&cmd.Storage, log.LogEventFromContext(ctx))
+	stg, err := util.StorageFromConfig(&cmd.Storage, a.brief.Me, log.LogEventFromContext(ctx))
 	if err != nil {
 		err = errors.Wrap(err, "storage from config")
 		return