From 086c93d72e241f9aeca4f1569ebac90498e36b6f Mon Sep 17 00:00:00 2001
From: Jiaqiang Huang <lzzziver@gmail.com>
Date: Fri, 20 Dec 2024 19:30:21 +0800
Subject: [PATCH] dm: delete file ignore blob file not exist (#11913)

close pingcap/tiflow#11912
---
 dm/pkg/storage/utils.go | 21 +++++++++++++++++----
 go.mod                  |  2 +-
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/dm/pkg/storage/utils.go b/dm/pkg/storage/utils.go
index 065c2f0c6ee..3a77d93a72f 100644
--- a/dm/pkg/storage/utils.go
+++ b/dm/pkg/storage/utils.go
@@ -21,9 +21,11 @@ import (
 	"strings"
 
 	gstorage "cloud.google.com/go/storage"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/pingcap/errors"
+	backuppb "github.com/pingcap/kvproto/pkg/brpb"
 	bstorage "github.com/pingcap/tidb/br/pkg/storage"
 )
 
@@ -133,25 +135,36 @@ func CollectDirFiles(ctx context.Context, dir string, storage bstorage.ExternalS
 // RemoveAll remove files in dir.
 func RemoveAll(ctx context.Context, dir string, storage bstorage.ExternalStorage) error {
 	var err error
+	var backend *backuppb.StorageBackend
 	if storage == nil {
 		storage, err = CreateStorage(ctx, dir)
 		if err != nil {
 			return err
 		}
+		backend, err = bstorage.ParseBackend(dir, nil)
+		if err != nil {
+			return err
+		}
 	}
 
 	err = storage.WalkDir(ctx, &bstorage.WalkOption{}, func(filePath string, size int64) error {
 		err2 := storage.DeleteFile(ctx, filePath)
-		if errors.Cause(err2) == gstorage.ErrObjectNotExist {
-			// ignore not exist error when we delete files
+		// ignore not exist error when we delete files and backend is gcs/azure blob storage
+		if backend != nil && backend.GetGcs() != nil && errors.Cause(err2) == gstorage.ErrObjectNotExist {
+			return nil
+		}
+		if backend != nil && backend.GetAzureBlobStorage() != nil && bloberror.HasCode(err2, bloberror.BlobNotFound) {
 			return nil
 		}
 		return err2
 	})
 	if err == nil {
 		err = storage.DeleteFile(ctx, "")
-		if errors.Cause(err) == gstorage.ErrObjectNotExist {
-			// ignore not exist error when we delete files
+		// ignore not exist error when we delete files and backend is gcs/azure blob storage
+		if backend != nil && backend.GetGcs() != nil && errors.Cause(err) == gstorage.ErrObjectNotExist {
+			return nil
+		}
+		if backend != nil && backend.GetAzureBlobStorage() != nil && bloberror.HasCode(err, bloberror.BlobNotFound) {
 			return nil
 		}
 	}
diff --git a/go.mod b/go.mod
index 0b0157a3507..929e90c020e 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,7 @@ go 1.23
 
 require (
 	cloud.google.com/go/storage v1.39.1
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
 	github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/IBM/sarama v1.41.2
@@ -129,7 +130,6 @@ require (
 
 require (
 	cloud.google.com/go/kms v1.15.8 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect