diff --git a/backend/plugins/customize/api/csv.go b/backend/plugins/customize/api/csv.go index fa9ae053a22..51ea626c8f6 100644 --- a/backend/plugins/customize/api/csv.go +++ b/backend/plugins/customize/api/csv.go @@ -34,6 +34,7 @@ const maxMemory = 32 << 20 // 32 MB // @Accept multipart/form-data // @Param boardId formData string true "the ID of the board" // @Param boardName formData string true "the name of the board" +// @Param incremental formData bool false "whether to import incrementally" // @Param file formData file true "select file to upload" // @Produce json // @Success 200 @@ -47,6 +48,10 @@ func (h *Handlers) ImportIssue(input *plugin.ApiResourceInput) (*plugin.ApiResou } // nolint defer file.Close() + incremental := false + if input.Request.FormValue("incremental") == "true" { + incremental = true + } boardId := strings.TrimSpace(input.Request.FormValue("boardId")) if boardId == "" { return nil, errors.BadInput.New("empty boardId") @@ -59,7 +64,7 @@ func (h *Handlers) ImportIssue(input *plugin.ApiResourceInput) (*plugin.ApiResou if err != nil { return nil, err } - return nil, h.svc.ImportIssue(boardId, file) + return nil, h.svc.ImportIssue(boardId, file, incremental) } // ImportIssueCommit accepts a CSV file, parses and saves it to the database @@ -94,6 +99,7 @@ func (h *Handlers) ImportIssueCommit(input *plugin.ApiResourceInput) (*plugin.Ap // @Tags plugins/customize // @Accept multipart/form-data // @Param boardId formData string true "the ID of the board" +// @Param incremental formData bool false "whether to import incrementally" // @Param file formData file true "select file to upload" // @Produce json // @Success 200 @@ -111,7 +117,11 @@ func (h *Handlers) ImportIssueRepoCommit(input *plugin.ApiResourceInput) (*plugi if boardId == "" { return nil, errors.Default.New("empty boardId") } - return nil, h.svc.ImportIssueRepoCommit(boardId, file) + incremental := false + if input.Request.FormValue("incremental") == "true" { + incremental = true + } + return nil, h.svc.ImportIssueRepoCommit(boardId, file, incremental) } func (h *Handlers) extractFile(input *plugin.ApiResourceInput) (io.ReadCloser, errors.Error) { diff --git a/backend/plugins/customize/e2e/import_issue_commits_test.go b/backend/plugins/customize/e2e/import_issue_commits_test.go index 3d1aeb64f36..f997d81df32 100644 --- a/backend/plugins/customize/e2e/import_issue_commits_test.go +++ b/backend/plugins/customize/e2e/import_issue_commits_test.go @@ -41,13 +41,13 @@ func TestImportIssueCommitDataFlow(t *testing.T) { } defer f.Close() // import data - err := svc.ImportIssueCommit(`{"ConnectionId":1,"BoardId":8}`, f) + err := svc.ImportIssueCommit(`csv-board`, f) if err != nil { t.Fatal(err) } dataflowTester.VerifyTableWithRawData( crossdomain.IssueCommit{}, - "snapshot_tables/issue_commits.csv", + "snapshot_tables/issue_commits_from_import_issue_commit.csv", []string{ "issue_id", "commit_sha", diff --git a/backend/plugins/customize/e2e/import_issue_repo_commits_test.go b/backend/plugins/customize/e2e/import_issue_repo_commits_test.go index 2d6041b1149..de1cca5ab58 100644 --- a/backend/plugins/customize/e2e/import_issue_repo_commits_test.go +++ b/backend/plugins/customize/e2e/import_issue_repo_commits_test.go @@ -18,12 +18,14 @@ limitations under the License. package e2e import ( + "os" + "testing" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/helpers/e2ehelper" "github.com/apache/incubator-devlake/plugins/customize/impl" "github.com/apache/incubator-devlake/plugins/customize/service" - "os" - "testing" ) func TestImportIssueRepoCommitDataFlow(t *testing.T) { @@ -33,18 +35,35 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) { // create tables `issue_repo_commits` and `issue_commits` dataflowTester.FlushTabler(&crossdomain.IssueRepoCommit{}) dataflowTester.FlushTabler(&crossdomain.IssueCommit{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + + dataflowTester.ImportCsvIntoTabler("raw_tables/issue_repo_commits_original.csv", &crossdomain.IssueRepoCommit{}) + dataflowTester.ImportCsvIntoTabler("raw_tables/board_issues.csv", &ticket.BoardIssue{}) + svc := service.NewService(dataflowTester.Dal) - f, err1 := os.Open("raw_tables/issue_repo_commits.csv") + issueRepoCommitsFile, err1 := os.Open("raw_tables/issue_repo_commits.csv") if err1 != nil { t.Fatal(err1) } - defer f.Close() + defer issueRepoCommitsFile.Close() // import data - err := svc.ImportIssueRepoCommit(`{"ConnectionId":1,"BoardId":8}`, f) + err := svc.ImportIssueRepoCommit("csv-board", issueRepoCommitsFile, false) if err != nil { t.Fatal(err) } + + // import data incrementally + issueRepoCommitsIncrementalFile, err2 := os.Open("raw_tables/issue_repo_commits_incremental.csv") + if err2 != nil { + t.Fatal(err2) + } + defer issueRepoCommitsIncrementalFile.Close() + err = svc.ImportIssueRepoCommit("csv-board", issueRepoCommitsIncrementalFile, true) + if err != nil { + t.Fatal(err) + } + dataflowTester.VerifyTableWithRawData( crossdomain.IssueRepoCommit{}, "snapshot_tables/issue_repo_commits.csv", @@ -58,7 +77,7 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) { }) dataflowTester.VerifyTableWithRawData( crossdomain.IssueCommit{}, - "snapshot_tables/issue_commits.csv", + "snapshot_tables/issue_commits_from_import_issue_repo_commit.csv", []string{ "issue_id", "commit_sha", diff --git a/backend/plugins/customize/e2e/import_issues_test.go b/backend/plugins/customize/e2e/import_issues_test.go index 8ed87803807..aa875eb92a5 100644 --- a/backend/plugins/customize/e2e/import_issues_test.go +++ b/backend/plugins/customize/e2e/import_issues_test.go @@ -89,21 +89,37 @@ func TestImportIssueDataFlow(t *testing.T) { t.Fatal(err1) } defer issueFile.Close() - err = svc.ImportIssue("csv-board", issueFile) + err = svc.ImportIssue("csv-board", issueFile, false) if err != nil { t.Fatal(err) } - - issueFile2, err2 := os.Open("raw_tables/issues_input2.csv") + issueAppendToFile1, err2 := os.Open("raw_tables/issues_input_incremental.csv") if err2 != nil { t.Fatal(err2) } + defer issueAppendToFile1.Close() + err = svc.ImportIssue("csv-board", issueAppendToFile1, true) + if err != nil { + t.Fatal(err) + } + issueFile2, err3 := os.Open("raw_tables/issues_input2.csv") + if err3 != nil { + t.Fatal(err3) + } defer issueFile2.Close() - err = svc.ImportIssue("csv-board2", issueFile2) + err = svc.ImportIssue("csv-board2", issueFile2, false) + if err != nil { + t.Fatal(err) + } + issueToOverwriteFile2, err4 := os.Open("raw_tables/issues_input2_overwrite.csv") + if err4 != nil { + t.Fatal(err4) + } + defer issueToOverwriteFile2.Close() + err = svc.ImportIssue("csv-board2", issueToOverwriteFile2, false) if err != nil { t.Fatal(err) } - dataflowTester.VerifyTableWithRawData( ticket.Issue{}, "snapshot_tables/issues_output.csv", diff --git a/backend/plugins/customize/e2e/raw_tables/board_issues.csv b/backend/plugins/customize/e2e/raw_tables/board_issues.csv new file mode 100644 index 00000000000..c88c197edec --- /dev/null +++ b/backend/plugins/customize/e2e/raw_tables/board_issues.csv @@ -0,0 +1,16 @@ +board_id,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +csv-board,csv:1,,,0, +csv-board,csv:10,,,0, +csv-board,csv:11,,,0, +csv-board,csv:12,,,0, +csv-board,csv:13,,,0, +csv-board,csv:14,,,0, +csv-board2,jira:JiraIssue:1:10063,,,0, +csv-board2,jira:JiraIssue:1:10064,,,0, +csv-board2,jira:JiraIssue:1:10065,,,0, +csv-board2,jira:JiraIssue:1:10066,,,0, +csv-board2,jira:JiraIssue:1:10139,,,0, +csv-board2,jira:JiraIssue:1:10145,,,0, +csv-board2,jira:JiraIssue:1:10159,,,0, +csv-board2,jira:JiraIssue:1:10202,,,0, +csv-board2,jira:JiraIssue:1:10203,,,0, diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv index b8a1314609c..477b47d7e11 100644 --- a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv +++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv @@ -11,15 +11,3 @@ csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/ csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake -jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake -jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo -jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co -jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_incremental.csv b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_incremental.csv new file mode 100644 index 00000000000..187a87afda3 --- /dev/null +++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_incremental.csv @@ -0,0 +1,13 @@ +issue_id,repo_url,commit_sha,host,namespace,repo_name +jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake +jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake +jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo +jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co +jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv new file mode 100644 index 00000000000..2e15ff01bb6 --- /dev/null +++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv @@ -0,0 +1,3 @@ +issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_removed,commit_id_should_be_removed,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, +jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_remained,commit_id_should_be_remained,example.com,PROJECTNAME,ui_jira,,,, diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input.csv b/backend/plugins/customize/e2e/raw_tables/issues_input.csv index ac05acd7623..32e72055f8f 100644 --- a/backend/plugins/customize/e2e/raw_tables/issues_input.csv +++ b/backend/plugins/customize/e2e/raw_tables/issues_input.csv @@ -2,6 +2,3 @@ id,url,issue_key,title,original_type,original_status,created_date,resolution_dat csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue test011,REQUIREMENT,new,2022-08-10 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL -csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,12,issue test012,REQUIREMENT,new,2022-08-11 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL -csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL -csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3" diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input2.csv b/backend/plugins/customize/e2e/raw_tables/issues_input2.csv index 215bc651d49..b7b08fb7787 100644 --- a/backend/plugins/customize/e2e/raw_tables/issues_input2.csv +++ b/backend/plugins/customize/e2e/raw_tables/issues_input2.csv @@ -1,3 +1,3 @@ id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels -csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL -csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds +csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL +csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3" diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input2_overwrite.csv b/backend/plugins/customize/e2e/raw_tables/issues_input2_overwrite.csv new file mode 100644 index 00000000000..549fefcae66 --- /dev/null +++ b/backend/plugins/customize/e2e/raw_tables/issues_input2_overwrite.csv @@ -0,0 +1,3 @@ +id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels +csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL +csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue title edited,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input_incremental.csv b/backend/plugins/customize/e2e/raw_tables/issues_input_incremental.csv new file mode 100644 index 00000000000..71ca994db7c --- /dev/null +++ b/backend/plugins/customize/e2e/raw_tables/issues_input_incremental.csv @@ -0,0 +1,4 @@ +id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels +csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,12,issue test012,REQUIREMENT,new,2022-08-11 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL +csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL +csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3" diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv deleted file mode 100644 index 97b875af10b..00000000000 --- a/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv +++ /dev/null @@ -1,25 +0,0 @@ -issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,, diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_commit.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_commit.csv new file mode 100644 index 00000000000..142fd78a096 --- /dev/null +++ b/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_commit.csv @@ -0,0 +1,25 @@ +issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,, +csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,, +csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,, +csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,, +csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,, +csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,, +csv:13,8993c04249e9d549e8950daec86717548c53c423,csv-board,,, +csv:14,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,, +csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,, +csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,, +csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,, +csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,, +jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,, +jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,, +jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,, +jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,, +jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,, +jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,csv-board,,, +jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,, +jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,, +jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,, +jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,, +jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,, +jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,, diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_repo_commit.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_repo_commit.csv new file mode 100644 index 00000000000..142fd78a096 --- /dev/null +++ b/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_repo_commit.csv @@ -0,0 +1,25 @@ +issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,, +csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,, +csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,, +csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,, +csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,, +csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,, +csv:13,8993c04249e9d549e8950daec86717548c53c423,csv-board,,, +csv:14,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,, +csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,, +csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,, +csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,, +csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,, +jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,, +jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,, +jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,, +jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,, +jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,, +jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,csv-board,,, +jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,, +jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,, +jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,, +jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,, +jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,, +jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,, diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv index d7523c026db..72e9ce73a7f 100644 --- a/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv +++ b/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv @@ -1,5 +1,5 @@ issue_id,label_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark csv:10,hello worlds,csv-board2,,0, -csv:14,label1,csv-board,,0, -csv:14,label2,csv-board,,0, -csv:14,label3,csv-board,,0, +csv:14,label1,csv-board2,,0, +csv:14,label2,csv-board2,,0, +csv:14,label3,csv-board2,,0, diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv index 311a9367861..b9e43abf052 100644 --- a/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv +++ b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv @@ -1,25 +1,26 @@ issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:10,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:11,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:12,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, +csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:10,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,csv-board,,, +csv:11,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,csv-board,,, +csv:12,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,csv-board,,, +csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:13,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,csv-board,,, +jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,csv-board,,, +jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_remained,commit_id_should_be_remained,example.com,PROJECTNAME,ui_jira,,,, +jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,csv-board,,, +jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,csv-board,,, +jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,csv-board,,, +jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,csv-board,,, +jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,csv-board,,, +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,csv-board,,, +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,csv-board,,, +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,csv-board,,, +jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,csv-board,,, +jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,csv-board,,, +jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,csv-board,,, diff --git a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv index 4e3528e1a3f..43d5220716a 100644 --- a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv +++ b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv @@ -1,7 +1,7 @@ id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,original_project,x_varchar,x_text,x_time,x_float,x_int,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,,tgp,,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,, -csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue test007,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,, +csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue title edited,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,, csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,, csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,, -csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,, -csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board,,, +csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board2,,, +csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board2,,, diff --git a/backend/plugins/customize/service/service.go b/backend/plugins/customize/service/service.go index a5187c28f58..6586dd13a9e 100644 --- a/backend/plugins/customize/service/service.go +++ b/backend/plugins/customize/service/service.go @@ -154,32 +154,33 @@ func (s *Service) getCustomizedFields(table string) ([]models.CustomizedField, e // ImportIssue import csv file to the table `issues`, and create relations to boards // issue could exist in multiple boards, so we should only delete an old records when it doesn't belong to another board -func (s *Service) ImportIssue(boardId string, file io.ReadCloser) errors.Error { - err := s.dal.Delete( - &ticket.Issue{}, - dal.Where("id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), - ) - if err != nil { - return err - } +func (s *Service) ImportIssue(boardId string, file io.ReadCloser, incremental bool) errors.Error { + if !incremental { + err := s.dal.Delete( + &ticket.Issue{}, + dal.Where("id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), + ) + if err != nil { + return err + } - err = s.dal.Delete( - &ticket.IssueLabel{}, - dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), - ) - if err != nil { - return err - } + err = s.dal.Delete( + &ticket.IssueLabel{}, + dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), + ) + if err != nil { + return err + } - err = s.dal.Delete( - &ticket.BoardIssue{}, - dal.Where("board_id = ?", boardId), - ) - if err != nil { - return err + err = s.dal.Delete( + &ticket.BoardIssue{}, + dal.Where("board_id = ?", boardId), + ) + if err != nil { + return err + } } - - return s.importCSV(file, boardId, s.issueHandlerFactory(boardId)) + return s.importCSV(file, boardId, s.issueHandlerFactory(boardId, incremental)) } // SaveBoard make sure the board exists in table `boards` @@ -206,21 +207,23 @@ func (s *Service) ImportIssueCommit(boardId string, file io.ReadCloser) errors.E } // ImportIssueRepoCommit imports data to the table `issue_repo_commits` and `issue_commits` -func (s *Service) ImportIssueRepoCommit(boardId string, file io.ReadCloser) errors.Error { - // delete old records of the table `issue_repo_commit` and `issue_commit` - err := s.dal.Delete( - &crossdomain.IssueRepoCommit{}, - dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), - ) - if err != nil { - return err - } - err = s.dal.Delete( - &crossdomain.IssueCommit{}, - dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), - ) - if err != nil { - return err +func (s *Service) ImportIssueRepoCommit(boardId string, file io.ReadCloser, incremental bool) errors.Error { + if !incremental { + // delete old records of the table `issue_repo_commit` and `issue_commit` + err := s.dal.Delete( + &crossdomain.IssueRepoCommit{}, + dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), + ) + if err != nil { + return err + } + err = s.dal.Delete( + &crossdomain.IssueCommit{}, + dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), + ) + if err != nil { + return err + } } return s.importCSV(file, boardId, s.issueRepoCommitHandler) } @@ -260,7 +263,7 @@ func (s *Service) importCSV(file io.ReadCloser, rawDataParams string, recordHand } // issueHandlerFactory returns a handler that save record into `issues`, `board_issues` and `issue_labels` table -func (s *Service) issueHandlerFactory(boardId string) func(record map[string]interface{}) errors.Error { +func (s *Service) issueHandlerFactory(boardId string, incremental bool) func(record map[string]interface{}) errors.Error { return func(record map[string]interface{}) errors.Error { var err errors.Error var id string