feat: support incremental import for /plugins/customize/csvfiles/issu…

…es.csv

[Feature][Customize] Add Support for Incremental CSV Upload in the Customize Plugin #8216

backend/plugins/customize/api/csv.go

@@ -34,6 +34,7 @@ const maxMemory = 32 << 20 // 32 MB
 // @Accept       multipart/form-data
 // @Param        boardId formData string true "the ID of the board"
 // @Param        boardName formData string true "the name of the board"
+// @Param        incremental formData bool false "whether to import incrementally"
 // @Param        file formData file true "select file to upload"
 // @Produce      json
 // @Success      200
@@ -47,6 +48,10 @@ func (h *Handlers) ImportIssue(input *plugin.ApiResourceInput) (*plugin.ApiResou
 	}
 	// nolint
 	defer file.Close()
+	incremental := false
+	if input.Request.FormValue("incremental") == "true" {
+		incremental = true
+	}
 	boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
 	if boardId == "" {
 		return nil, errors.BadInput.New("empty boardId")
@@ -59,7 +64,7 @@ func (h *Handlers) ImportIssue(input *plugin.ApiResourceInput) (*plugin.ApiResou
 	if err != nil {
 		return nil, err
 	}
-	return nil, h.svc.ImportIssue(boardId, file)
+	return nil, h.svc.ImportIssue(boardId, file, incremental)
 }
 
 // ImportIssueCommit accepts a CSV file, parses and saves it to the database

backend/plugins/customize/e2e/import_issues_test.go

@@ -89,21 +89,37 @@ func TestImportIssueDataFlow(t *testing.T) {
 		t.Fatal(err1)
 	}
 	defer issueFile.Close()
-	err = svc.ImportIssue("csv-board", issueFile)
+	err = svc.ImportIssue("csv-board", issueFile, false)
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	issueFile2, err2 := os.Open("raw_tables/issues_input2.csv")
+	issueAppendToFile1, err2 := os.Open("raw_tables/issues_input_incremental.csv")
 	if err2 != nil {
 		t.Fatal(err2)
 	}
+	defer issueAppendToFile1.Close()
+	err = svc.ImportIssue("csv-board", issueAppendToFile1, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	issueFile2, err3 := os.Open("raw_tables/issues_input2.csv")
+	if err3 != nil {
+		t.Fatal(err3)
+	}
 	defer issueFile2.Close()
-	err = svc.ImportIssue("csv-board2", issueFile2)
+	err = svc.ImportIssue("csv-board2", issueFile2, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	issueToOverwriteFile2, err4 := os.Open("raw_tables/issues_input2_overwrite.csv")
+	if err4 != nil {
+		t.Fatal(err4)
+	}
+	defer issueToOverwriteFile2.Close()
+	err = svc.ImportIssue("csv-board2", issueToOverwriteFile2, false)
 	if err != nil {
 		t.Fatal(err)
 	}
-
 	dataflowTester.VerifyTableWithRawData(
 		ticket.Issue{},
 		"snapshot_tables/issues_output.csv",

backend/plugins/customize/e2e/raw_tables/issues_input.csv

@@ -2,6 +2,3 @@ id,url,issue_key,title,original_type,original_status,created_date,resolution_dat
 csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL
 csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds
 csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue test011,REQUIREMENT,new,2022-08-10 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
-csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,12,issue test012,REQUIREMENT,new,2022-08-11 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
-csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
-csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3"

backend/plugins/customize/e2e/raw_tables/issues_input2.csv

@@ -1,3 +1,3 @@
 id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
-csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds
+csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
+csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3"

backend/plugins/customize/e2e/raw_tables/issues_input2_overwrite.csv

@@ -0,0 +1,3 @@
+id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
+csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue title edited,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds

backend/plugins/customize/e2e/raw_tables/issues_input_incremental.csv

@@ -0,0 +1,4 @@
+id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
+csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,12,issue test012,REQUIREMENT,new,2022-08-11 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
+csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
+csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3"

backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv

@@ -1,5 +1,5 @@
 issue_id,label_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
 csv:10,hello worlds,csv-board2,,0,
-csv:14,label1,csv-board,,0,
-csv:14,label2,csv-board,,0,
-csv:14,label3,csv-board,,0,
+csv:14,label1,csv-board2,,0,
+csv:14,label2,csv-board2,,0,
+csv:14,label3,csv-board2,,0,

backend/plugins/customize/e2e/snapshot_tables/issues_output.csv

@@ -1,7 +1,7 @@
 id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,original_project,x_varchar,x_text,x_time,x_float,x_int,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
 csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,,tgp,,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,,
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue test007,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,,
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue title edited,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,,
 csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
 csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
-csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
-csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board,,,
+csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board2,,,
+csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board2,,,

backend/plugins/customize/service/service.go

@@ -154,32 +154,33 @@ func (s *Service) getCustomizedFields(table string) ([]models.CustomizedField, e
 
 // ImportIssue import csv file to the table `issues`, and create relations to boards
 // issue could exist in multiple boards, so we should only delete an old records when it doesn't belong to another board
-func (s *Service) ImportIssue(boardId string, file io.ReadCloser) errors.Error {
-	err := s.dal.Delete(
-		&ticket.Issue{},
-		dal.Where("id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
-	)
-	if err != nil {
-		return err
-	}
+func (s *Service) ImportIssue(boardId string, file io.ReadCloser, incremental bool) errors.Error {
+	if !incremental {
+		err := s.dal.Delete(
+			&ticket.Issue{},
+			dal.Where("id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
+		)
+		if err != nil {
+			return err
+		}
 
-	err = s.dal.Delete(
-		&ticket.IssueLabel{},
-		dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
-	)
-	if err != nil {
-		return err
-	}
+		err = s.dal.Delete(
+			&ticket.IssueLabel{},
+			dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
+		)
+		if err != nil {
+			return err
+		}
 
-	err = s.dal.Delete(
-		&ticket.BoardIssue{},
-		dal.Where("board_id = ?", boardId),
-	)
-	if err != nil {
-		return err
+		err = s.dal.Delete(
+			&ticket.BoardIssue{},
+			dal.Where("board_id = ?", boardId),
+		)
+		if err != nil {
+			return err
+		}
 	}
-
-	return s.importCSV(file, boardId, s.issueHandlerFactory(boardId))
+	return s.importCSV(file, boardId, s.issueHandlerFactory(boardId, incremental))
 }
 
 // SaveBoard make sure the board exists in table `boards`
@@ -260,7 +261,7 @@ func (s *Service) importCSV(file io.ReadCloser, rawDataParams string, recordHand
 }
 
 // issueHandlerFactory returns a handler that save record into `issues`, `board_issues` and `issue_labels` table
-func (s *Service) issueHandlerFactory(boardId string) func(record map[string]interface{}) errors.Error {
+func (s *Service) issueHandlerFactory(boardId string, incremental bool) func(record map[string]interface{}) errors.Error {
 	return func(record map[string]interface{}) errors.Error {
 		var err errors.Error
 		var id string