From 293309244be6212abf896a32c24cf25009c6f692 Mon Sep 17 00:00:00 2001 From: Dmytro Zghoba Date: Mon, 9 Oct 2023 09:16:31 +0300 Subject: [PATCH] add --dry-run --- cmd/pbm/delete.go | 104 +++++++++++++++++++------ cmd/pbm/main.go | 14 ++-- internal/backup/delete.go | 58 +++++++++----- internal/backup/{cache.go => query.go} | 2 +- sdk/util.go | 28 +++++++ 5 files changed, 157 insertions(+), 49 deletions(-) rename internal/backup/{cache.go => query.go} (99%) diff --git a/cmd/pbm/delete.go b/cmd/pbm/delete.go index 8ba9d538e..1130844df 100644 --- a/cmd/pbm/delete.go +++ b/cmd/pbm/delete.go @@ -25,7 +25,8 @@ type deleteBcpOpts struct { name string olderThan string bcpType string - force bool + dryRun bool + yes bool } func deleteBackup( @@ -42,30 +43,18 @@ func deleteBackup( return nil, errors.New("either --name or --older-than should be set") } - if !d.force { - if err := askConfirmation("Are you sure you want to delete backup(s)?"); err != nil { - if errors.Is(err, errUserCanceled) { - return outMsg{err.Error()}, nil - } - return nil, err - } - } - var cid sdk.CommandID var err error if d.name != "" { - cid, err = pbm.DeleteBackupByName(ctx, d.name) - } else { // d.olderThan != "" - var ts primitive.Timestamp - ts, err = parseOlderThan(d.olderThan) - if err != nil { - return nil, errors.Wrap(err, "parse --older-than") - } - - cid, err = pbm.DeleteBackupBefore(ctx, ts) + cid, err = deleteBackupByName(ctx, pbm, d) + } else { + cid, err = deleteManyBackup(ctx, pbm, d) } if err != nil { - return nil, errors.Wrap(err, "schedule delete") + if errors.Is(err, errUserCanceled) { + return outMsg{err.Error()}, nil + } + return nil, err } if outf != outText { @@ -75,9 +64,80 @@ func deleteBackup( return waitForDelete(ctx, conn, pbm, cid) } +func deleteBackupByName(ctx context.Context, pbm sdk.Client, d *deleteBcpOpts) (sdk.CommandID, error) { + bcp, err := pbm.GetBackupByName(ctx, d.name) + if err != nil { + return sdk.NoOpID, errors.Wrap(err, "get backup metadata") + } + err = sdk.CanDeleteBackup(ctx, pbm, bcp) + if err != nil { + return sdk.NoOpID, errors.Wrap(err, "backup cannot be deleted") + } + + fmt.Println(shortBcpInfo(bcp)) + + if d.dryRun { + return sdk.NoOpID, nil + } + if !d.yes { + err := askConfirmation("Are you sure you want to delete backup?") + if err != nil { + return sdk.NoOpID, err + } + } + + cid, err := pbm.DeleteBackupByName(ctx, d.name) + return cid, errors.Wrap(err, "schedule delete") +} + +func deleteManyBackup(ctx context.Context, pbm sdk.Client, d *deleteBcpOpts) (sdk.CommandID, error) { + var ts primitive.Timestamp + ts, err := parseOlderThan(d.olderThan) + if err != nil { + return sdk.NoOpID, errors.Wrap(err, "parse --older-than") + } + + bcpType := sdk.ParseBackupType(d.bcpType) + + backups, err := sdk.ListDeleteBackupBefore(ctx, pbm, ts, bcpType) + if err != nil { + return sdk.NoOpID, errors.Wrap(err, "fetch backup list") + } + + for i := range backups { + fmt.Println(shortBcpInfo(&backups[i])) + } + + if d.dryRun { + return sdk.NoOpID, nil + } + if !d.yes { + if err := askConfirmation("Are you sure you want to delete backups?"); err != nil { + return sdk.NoOpID, err + } + } + + cid, err := pbm.DeleteBackupBefore(ctx, ts) + return cid, errors.Wrap(err, "schedule delete") +} + +func shortBcpInfo(bcp *sdk.BackupMetadata) string { + size := fmtSize(bcp.Size) + + t := string(bcp.Type) + if bcp.Type == sdk.LogicalBackup && len(bcp.Namespaces) != 0 { + t += ", selective" + } else if bcp.Type == defs.IncrementalBackup && bcp.SrcBackup == "" { + t += ", base" + } + + restoreTime := time.Unix(int64(bcp.LastWriteTS.T), 0).UTC().Format(time.RFC3339) + return fmt.Sprintf("%q [size: %s type: <%s>, restore time: %s]", bcp.Name, size, t, restoreTime) +} + type deletePitrOpts struct { olderThan string - force bool + yes bool all bool } @@ -95,7 +155,7 @@ func deletePITR( return nil, errors.New("either --older-than or --all should be set") } - if !d.force { + if !d.yes { q := "Are you sure you want to delete chunks?" if d.all { q = "Are you sure you want to delete ALL chunks?" diff --git a/cmd/pbm/main.go b/cmd/pbm/main.go index 580210646..fed289083 100644 --- a/cmd/pbm/main.go +++ b/cmd/pbm/main.go @@ -247,10 +247,12 @@ func main() { ) deleteBcpCmd.Flag("yes", "Don't ask confirmation"). Short('y'). - BoolVar(&deleteBcp.force) - deleteBcpCmd.Flag("force", "Force. Don't ask confirmation"). + BoolVar(&deleteBcp.yes) + deleteBcpCmd.Flag("force", "Don't ask confirmation (deprecated)"). Short('f'). - BoolVar(&deleteBcp.force) + BoolVar(&deleteBcp.yes) + deleteBcpCmd.Flag("dry-run", "Report but do not delete"). + BoolVar(&deleteBcp.dryRun) deletePitrCmd := pbmCmd.Command("delete-pitr", "Delete PITR chunks") deletePitr := deletePitrOpts{} @@ -264,10 +266,10 @@ func main() { BoolVar(&deletePitr.all) deletePitrCmd.Flag("yes", "Don't ask confirmation"). Short('y'). - BoolVar(&deletePitr.force) - deletePitrCmd.Flag("force", "Force. Don't ask confirmation"). + BoolVar(&deletePitr.yes) + deletePitrCmd.Flag("force", "Don't ask confirmation (deprecated)"). Short('f'). - BoolVar(&deletePitr.force) + BoolVar(&deletePitr.yes) cleanupCmd := pbmCmd.Command("cleanup", "Delete Backups and PITR chunks") cleanupOpts := cleanupOptions{} diff --git a/internal/backup/delete.go b/internal/backup/delete.go index 9ee7017be..a1a5df119 100644 --- a/internal/backup/delete.go +++ b/internal/backup/delete.go @@ -24,7 +24,6 @@ var ( errBackupInProgress = errors.New("backup is in progress") errSourceForIncremental = errors.New("the backup required for following incremental") errBaseForPITR = errors.New("unable to delete the last backup while PITR is enabled") - errNothing = errors.New("nothing") ) type CleanupInfo struct { @@ -67,11 +66,11 @@ func CanDeleteBackup(ctx context.Context, cc connect.Client, bcp *BackupMeta) er if bcp.Status.IsRunning() { return errBackupInProgress } - if !IsValidBaseSnapshot(bcp) { + if !isValidBaseSnapshot(bcp) { return nil } - isSource, err := IsSourceForIncremental(ctx, cc, bcp.Name) + isSource, err := isSourceForIncremental(ctx, cc, bcp.Name) if err != nil { return errors.Wrap(err, "check source incremental") } @@ -79,7 +78,7 @@ func CanDeleteBackup(ctx context.Context, cc connect.Client, bcp *BackupMeta) er return errSourceForIncremental } - required, err := IsRequiredForPITR(ctx, cc, bcp.LastWriteTS) + required, err := isRequiredForPITR(ctx, cc, bcp.LastWriteTS) if err != nil { return errors.Wrap(err, "check pitr requirements") } @@ -90,7 +89,7 @@ func CanDeleteBackup(ctx context.Context, cc connect.Client, bcp *BackupMeta) er return nil } -func IsSourceForIncremental(ctx context.Context, cc connect.Client, bcpName string) (bool, error) { +func isSourceForIncremental(ctx context.Context, cc connect.Client, bcpName string) (bool, error) { // check if there is an increment based on the backup f := bson.D{ {"src_backup", bcpName}, @@ -108,7 +107,7 @@ func IsSourceForIncremental(ctx context.Context, cc connect.Client, bcpName stri return true, nil } -func IsValidBaseSnapshot(bcp *BackupMeta) bool { +func isValidBaseSnapshot(bcp *BackupMeta) bool { if bcp.Status != defs.StatusDone { return false } @@ -122,18 +121,17 @@ func IsValidBaseSnapshot(bcp *BackupMeta) bool { return true } -func IsRequiredForPITR(ctx context.Context, cc connect.Client, lw primitive.Timestamp) (bool, error) { +func isRequiredForPITR(ctx context.Context, cc connect.Client, lw primitive.Timestamp) (bool, error) { enabled, oplogOnly, err := config.IsPITREnabled(ctx, cc) if err != nil { return false, err } - // the backup with restore time `lw` can be deleted only if it is not used by running PITR if !enabled || oplogOnly { return false, nil } - has, err := IsBaseSnapshotAfter(ctx, cc, lw) + has, err := HasBaseSnapshotAfter(ctx, cc, lw) if err != nil { return false, errors.Wrap(err, "check next base snapshot") } @@ -143,22 +141,18 @@ func IsRequiredForPITR(ctx context.Context, cc connect.Client, lw primitive.Time // DeleteOlderThan deletes backups which older than given Time func DeleteOlderThan(ctx context.Context, cc connect.Client, t time.Time, bcpType defs.BackupType) error { - info, err := MakeCleanupInfo(ctx, cc, primitive.Timestamp{T: uint32(t.Unix())}) + backups, err := ListDeleteBackupBefore(ctx, cc, primitive.Timestamp{T: uint32(t.Unix())}, bcpType) if err != nil { return err } - if len(info.Backups) == 0 { - return errNothing + if len(backups) == 0 { + return nil } stg, err := util.GetStorage(ctx, cc, log.LogEventFromContext(ctx)) if err != nil { return errors.Wrap(err, "get storage") } - backups := filterBackupsByType(info.Backups, bcpType) - if len(backups) == 0 { - return errNothing - } for i := range backups { bcp := &backups[i] @@ -177,6 +171,24 @@ func DeleteOlderThan(ctx context.Context, cc connect.Client, t time.Time, bcpTyp return nil } +func ListDeleteBackupBefore( + ctx context.Context, + cc connect.Client, + ts primitive.Timestamp, + bcpType defs.BackupType, +) ([]BackupMeta, error) { + info, err := MakeCleanupInfo(ctx, cc, ts) + if err != nil { + return nil, err + } + if len(info.Backups) == 0 { + return nil, nil + } + + backups := filterBackupsByType(info.Backups, bcpType) + return backups, nil +} + func filterBackupsByType(backups []BackupMeta, t defs.BackupType) []BackupMeta { rv := []BackupMeta{} @@ -203,7 +215,7 @@ func MakeCleanupInfo(ctx context.Context, conn connect.Client, ts primitive.Time exclude := true if l := len(backups) - 1; l != -1 && backups[l].LastWriteTS.T == ts.T { // there is a backup at the `ts` - if backups[l].Status == defs.StatusDone && !util.IsSelective(backups[l].Namespaces) { + if isValidBaseSnapshot(&backups[l]) { // it can be used to fully restore data to the `ts` state. // no need to exclude any base snapshot and chunks before the `ts` exclude = false @@ -232,7 +244,7 @@ func MakeCleanupInfo(ctx context.Context, conn connect.Client, ts primitive.Time // if there is no base snapshot after `ts` and PITR is running, // the last base snapshot before `ts` should be excluded. // otherwise, it is allowed to delete everything before `ts` - required, err := IsRequiredForPITR(ctx, conn, ts) + required, err := isRequiredForPITR(ctx, conn, ts) if err != nil { return CleanupInfo{}, err } @@ -305,6 +317,9 @@ func extractLastIncrementalChain( // lookup for the last incremental i := len(bcps) - 1 for ; i != -1; i-- { + if bcps[i].Status != defs.StatusDone { + continue + } if bcps[i].Type == defs.IncrementalBackup { break } @@ -314,7 +329,7 @@ func extractLastIncrementalChain( return bcps, nil } - isSource, err := IsSourceForIncremental(ctx, conn, bcps[i].Name) + isSource, err := isSourceForIncremental(ctx, conn, bcps[i].Name) if err != nil { return bcps, err } @@ -323,6 +338,9 @@ func extractLastIncrementalChain( } for base := bcps[i].Name; i != -1; i-- { + if bcps[i].Status != defs.StatusDone { + continue + } if bcps[i].Name != base { continue } @@ -343,7 +361,7 @@ func extractLastIncrementalChain( func findLastBaseSnapshotIndex(bcps []BackupMeta) int { for i := len(bcps) - 1; i != -1; i-- { - if IsValidBaseSnapshot(&bcps[i]) { + if isValidBaseSnapshot(&bcps[i]) { return i } } diff --git a/internal/backup/cache.go b/internal/backup/query.go similarity index 99% rename from internal/backup/cache.go rename to internal/backup/query.go index 58bb504c5..628fccd86 100644 --- a/internal/backup/cache.go +++ b/internal/backup/query.go @@ -284,7 +284,7 @@ func getRecentBackup( return b, errors.Wrap(err, "decode") } -func IsBaseSnapshotAfter(ctx context.Context, conn connect.Client, lw primitive.Timestamp) (bool, error) { +func HasBaseSnapshotAfter(ctx context.Context, conn connect.Client, lw primitive.Timestamp) (bool, error) { f := bson.D{ {"nss", nil}, {"type", bson.M{"$ne": defs.ExternalBackup}}, diff --git a/sdk/util.go b/sdk/util.go index fbb549ab0..c5c3b2fbc 100644 --- a/sdk/util.go +++ b/sdk/util.go @@ -6,6 +6,7 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "github.com/percona/percona-backup-mongodb/internal/backup" "github.com/percona/percona-backup-mongodb/internal/defs" "github.com/percona/percona-backup-mongodb/internal/errors" "github.com/percona/percona-backup-mongodb/internal/topo" @@ -13,6 +14,20 @@ import ( var errMissedClusterTime = errors.New("missed cluster time") +func ParseBackupType(s string) BackupType { + switch s { + case + string(PhysicalBackup), + string(ExternalBackup), + string(IncrementalBackup), + string(LogicalBackup), + string(SelectiveBackup): + return BackupType(s) + } + + return "" +} + func IsHeartbeatStale(clusterTime, other Timestamp) bool { return clusterTime.T >= other.T+defs.StaleFrameSec } @@ -28,3 +43,16 @@ func GetClusterTime(ctx context.Context, m *mongo.Client) (Timestamp, error) { return info.ClusterTime.ClusterTime, nil } + +func CanDeleteBackup(ctx context.Context, sc Client, bcp *BackupMetadata) error { + return backup.CanDeleteBackup(ctx, sc.(*clientImpl).conn, bcp) +} + +func ListDeleteBackupBefore( + ctx context.Context, + sc Client, + ts primitive.Timestamp, + bcpType BackupType, +) ([]BackupMetadata, error) { + return backup.ListDeleteBackupBefore(ctx, sc.(*clientImpl).conn, ts, bcpType) +}