diff --git a/backend/plugins/customize/api/csv.go b/backend/plugins/customize/api/csv.go index 472f448e8d0..51ea626c8f6 100644 --- a/backend/plugins/customize/api/csv.go +++ b/backend/plugins/customize/api/csv.go @@ -99,6 +99,7 @@ func (h *Handlers) ImportIssueCommit(input *plugin.ApiResourceInput) (*plugin.Ap // @Tags plugins/customize // @Accept multipart/form-data // @Param boardId formData string true "the ID of the board" +// @Param incremental formData bool false "whether to import incrementally" // @Param file formData file true "select file to upload" // @Produce json // @Success 200 @@ -116,7 +117,11 @@ func (h *Handlers) ImportIssueRepoCommit(input *plugin.ApiResourceInput) (*plugi if boardId == "" { return nil, errors.Default.New("empty boardId") } - return nil, h.svc.ImportIssueRepoCommit(boardId, file) + incremental := false + if input.Request.FormValue("incremental") == "true" { + incremental = true + } + return nil, h.svc.ImportIssueRepoCommit(boardId, file, incremental) } func (h *Handlers) extractFile(input *plugin.ApiResourceInput) (io.ReadCloser, errors.Error) { diff --git a/backend/plugins/customize/e2e/import_issue_repo_commits_test.go b/backend/plugins/customize/e2e/import_issue_repo_commits_test.go index 2d6041b1149..3e8d07ea621 100644 --- a/backend/plugins/customize/e2e/import_issue_repo_commits_test.go +++ b/backend/plugins/customize/e2e/import_issue_repo_commits_test.go @@ -18,12 +18,14 @@ limitations under the License. package e2e import ( + "os" + "testing" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/helpers/e2ehelper" "github.com/apache/incubator-devlake/plugins/customize/impl" "github.com/apache/incubator-devlake/plugins/customize/service" - "os" - "testing" ) func TestImportIssueRepoCommitDataFlow(t *testing.T) { @@ -33,6 +35,11 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) { // create tables `issue_repo_commits` and `issue_commits` dataflowTester.FlushTabler(&crossdomain.IssueRepoCommit{}) dataflowTester.FlushTabler(&crossdomain.IssueCommit{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + + dataflowTester.ImportCsvIntoTabler("raw_tables/issue_repo_commits_original.csv", &crossdomain.IssueRepoCommit{}) + dataflowTester.ImportCsvIntoTabler("raw_tables/board_issues.csv", &ticket.BoardIssue{}) + svc := service.NewService(dataflowTester.Dal) f, err1 := os.Open("raw_tables/issue_repo_commits.csv") @@ -41,10 +48,22 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) { } defer f.Close() // import data - err := svc.ImportIssueRepoCommit(`{"ConnectionId":1,"BoardId":8}`, f) + err := svc.ImportIssueRepoCommit("csv-board", f, false) if err != nil { t.Fatal(err) } + + // import data incrementally + f2, err2 := os.Open("raw_tables/issue_repo_commits2.csv") + if err2 != nil { + t.Fatal(err2) + } + defer f2.Close() + err = svc.ImportIssueRepoCommit("csv-board2", f2, true) + if err != nil { + t.Fatal(err) + } + dataflowTester.VerifyTableWithRawData( crossdomain.IssueRepoCommit{}, "snapshot_tables/issue_repo_commits.csv", diff --git a/backend/plugins/customize/e2e/raw_tables/board_issues.csv b/backend/plugins/customize/e2e/raw_tables/board_issues.csv new file mode 100644 index 00000000000..c88c197edec --- /dev/null +++ b/backend/plugins/customize/e2e/raw_tables/board_issues.csv @@ -0,0 +1,16 @@ +board_id,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +csv-board,csv:1,,,0, +csv-board,csv:10,,,0, +csv-board,csv:11,,,0, +csv-board,csv:12,,,0, +csv-board,csv:13,,,0, +csv-board,csv:14,,,0, +csv-board2,jira:JiraIssue:1:10063,,,0, +csv-board2,jira:JiraIssue:1:10064,,,0, +csv-board2,jira:JiraIssue:1:10065,,,0, +csv-board2,jira:JiraIssue:1:10066,,,0, +csv-board2,jira:JiraIssue:1:10139,,,0, +csv-board2,jira:JiraIssue:1:10145,,,0, +csv-board2,jira:JiraIssue:1:10159,,,0, +csv-board2,jira:JiraIssue:1:10202,,,0, +csv-board2,jira:JiraIssue:1:10203,,,0, diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv index b8a1314609c..477b47d7e11 100644 --- a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv +++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv @@ -11,15 +11,3 @@ csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/ csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake -jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake -jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo -jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co -jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira -jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits2.csv b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits2.csv new file mode 100644 index 00000000000..187a87afda3 --- /dev/null +++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits2.csv @@ -0,0 +1,13 @@ +issue_id,repo_url,commit_sha,host,namespace,repo_name +jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake +jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake +jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo +jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co +jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira +jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv new file mode 100644 index 00000000000..2e15ff01bb6 --- /dev/null +++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv @@ -0,0 +1,3 @@ +issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_removed,commit_id_should_be_removed,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, +jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_remained,commit_id_should_be_remained,example.com,PROJECTNAME,ui_jira,,,, diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv index 97b875af10b..c90d0fe8fbc 100644 --- a/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv +++ b/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv @@ -1,25 +1,25 @@ issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,, +csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,, +csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,, +csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,, +csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,, +csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,, +csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,, +csv:13,8993c04249e9d549e8950daec86717548c53c423,csv-board,,, +csv:14,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,, +csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,, +csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,, +csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,, +csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,, +jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board2,,, +jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board2,,, +jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board2,,, +jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board2,,, +jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board2,,, +jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,csv-board2,,, +jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board2,,, +jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board2,,, +jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board2,,, +jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board2,,, +jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,csv-board2,,, +jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board2,,, diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv index 311a9367861..43290a7691b 100644 --- a/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv +++ b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv @@ -1,25 +1,26 @@ issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:10,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:11,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:12,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:13,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, -jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,, +csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:10,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,csv-board,,, +csv:11,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,csv-board,,, +csv:12,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,csv-board,,, +csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:13,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,csv-board,,, +csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,csv-board,,, +jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,csv-board2,,, +jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_remained,commit_id_should_be_remained,example.com,PROJECTNAME,ui_jira,,,, +jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,csv-board2,,, +jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,csv-board2,,, +jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,csv-board2,,, +jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,csv-board2,,, +jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,csv-board2,,, +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,csv-board2,,, +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,csv-board2,,, +jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,csv-board2,,, +jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,csv-board2,,, +jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,csv-board2,,, +jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,csv-board2,,, diff --git a/backend/plugins/customize/service/service.go b/backend/plugins/customize/service/service.go index d15e06a1ddc..6586dd13a9e 100644 --- a/backend/plugins/customize/service/service.go +++ b/backend/plugins/customize/service/service.go @@ -207,21 +207,23 @@ func (s *Service) ImportIssueCommit(boardId string, file io.ReadCloser) errors.E } // ImportIssueRepoCommit imports data to the table `issue_repo_commits` and `issue_commits` -func (s *Service) ImportIssueRepoCommit(boardId string, file io.ReadCloser) errors.Error { - // delete old records of the table `issue_repo_commit` and `issue_commit` - err := s.dal.Delete( - &crossdomain.IssueRepoCommit{}, - dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), - ) - if err != nil { - return err - } - err = s.dal.Delete( - &crossdomain.IssueCommit{}, - dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), - ) - if err != nil { - return err +func (s *Service) ImportIssueRepoCommit(boardId string, file io.ReadCloser, incremental bool) errors.Error { + if !incremental { + // delete old records of the table `issue_repo_commit` and `issue_commit` + err := s.dal.Delete( + &crossdomain.IssueRepoCommit{}, + dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), + ) + if err != nil { + return err + } + err = s.dal.Delete( + &crossdomain.IssueCommit{}, + dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), + ) + if err != nil { + return err + } } return s.importCSV(file, boardId, s.issueRepoCommitHandler) }