Skip to content

Commit

Permalink
feat: support incremental import for/plugins/customize/csvfiles/issue…
Browse files Browse the repository at this point in the history 
…_repo_commits.csv

    [Feature][Customize] Add Support for Incremental CSV Upload in the Customize Plugin #8216
  • Loading branch information
@narrowizard
narrowizard committed Nov 25, 2024
1 parent 1dcb0ea commit 828f498
Show file tree
Hide file tree
Showing 9 changed files with 126 additions and 79 deletions.
7 changes: 6 additions & 1 deletion backend/plugins/customize/api/csv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ func (h *Handlers) ImportIssueCommit(input *plugin.ApiResourceInput) (*plugin.Ap
// @Tags plugins/customize
// @Accept multipart/form-data
// @Param boardId formData string true "the ID of the board"
// @Param incremental formData bool false "whether to import incrementally"
// @Param file formData file true "select file to upload"
// @Produce json
// @Success 200
Expand All @@ -116,7 +117,11 @@ func (h *Handlers) ImportIssueRepoCommit(input *plugin.ApiResourceInput) (*plugi
if boardId == "" {
return nil, errors.Default.New("empty boardId")
}
return nil, h.svc.ImportIssueRepoCommit(boardId, file)
incremental := false
if input.Request.FormValue("incremental") == "true" {
incremental = true
}
return nil, h.svc.ImportIssueRepoCommit(boardId, file, incremental)
}

func (h *Handlers) extractFile(input *plugin.ApiResourceInput) (io.ReadCloser, errors.Error) {
Expand Down
25 changes: 22 additions & 3 deletions backend/plugins/customize/e2e/import_issue_repo_commits_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ limitations under the License.
package e2e

import (
"os"
"testing"

"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/customize/impl"
"github.com/apache/incubator-devlake/plugins/customize/service"
"os"
"testing"
)

func TestImportIssueRepoCommitDataFlow(t *testing.T) {
Expand All @@ -33,6 +35,11 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) {
// create tables `issue_repo_commits` and `issue_commits`
dataflowTester.FlushTabler(&crossdomain.IssueRepoCommit{})
dataflowTester.FlushTabler(&crossdomain.IssueCommit{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})

dataflowTester.ImportCsvIntoTabler("raw_tables/issue_repo_commits_original.csv", &crossdomain.IssueRepoCommit{})
dataflowTester.ImportCsvIntoTabler("raw_tables/board_issues.csv", &ticket.BoardIssue{})

svc := service.NewService(dataflowTester.Dal)

f, err1 := os.Open("raw_tables/issue_repo_commits.csv")
Expand All @@ -41,10 +48,22 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) {
}
defer f.Close()
// import data
err := svc.ImportIssueRepoCommit(`{"ConnectionId":1,"BoardId":8}`, f)
err := svc.ImportIssueRepoCommit("csv-board", f, false)
if err != nil {
t.Fatal(err)
}

// import data incrementally
f2, err2 := os.Open("raw_tables/issue_repo_commits2.csv")
if err2 != nil {
t.Fatal(err2)
}
defer f2.Close()
err = svc.ImportIssueRepoCommit("csv-board2", f2, true)
if err != nil {
t.Fatal(err)
}

dataflowTester.VerifyTableWithRawData(
crossdomain.IssueRepoCommit{},
"snapshot_tables/issue_repo_commits.csv",
Expand Down
16 changes: 16 additions & 0 deletions backend/plugins/customize/e2e/raw_tables/board_issues.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
board_id,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
csv-board,csv:1,,,0,
csv-board,csv:10,,,0,
csv-board,csv:11,,,0,
csv-board,csv:12,,,0,
csv-board,csv:13,,,0,
csv-board,csv:14,,,0,
csv-board2,jira:JiraIssue:1:10063,,,0,
csv-board2,jira:JiraIssue:1:10064,,,0,
csv-board2,jira:JiraIssue:1:10065,,,0,
csv-board2,jira:JiraIssue:1:10066,,,0,
csv-board2,jira:JiraIssue:1:10139,,,0,
csv-board2,jira:JiraIssue:1:10145,,,0,
csv-board2,jira:JiraIssue:1:10159,,,0,
csv-board2,jira:JiraIssue:1:10202,,,0,
csv-board2,jira:JiraIssue:1:10203,,,0,
12 changes: 0 additions & 12 deletions backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,3 @@ csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake
jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake
jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo
jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co
jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
13 changes: 13 additions & 0 deletions backend/plugins/customize/e2e/raw_tables/issue_repo_commits2.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
issue_id,repo_url,commit_sha,host,namespace,repo_name
jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake
jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake
jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo
jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co
jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
3 changes: 3 additions & 0 deletions backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_removed,commit_id_should_be_removed,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_remained,commit_id_should_be_remained,example.com,PROJECTNAME,ui_jira,,,,
48 changes: 24 additions & 24 deletions backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:13,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:14,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,,
jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,,
csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,,
csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,,
csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,,
csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,,
csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,,
csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,,
csv:13,8993c04249e9d549e8950daec86717548c53c423,csv-board,,,
csv:14,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,,
csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,,
csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,,
csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,,
csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,,
jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board2,,,
jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board2,,,
jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board2,,,
jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board2,,,
jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board2,,,
jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,csv-board2,,,
jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board2,,,
jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board2,,,
jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board2,,,
jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board2,,,
jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,csv-board2,,,
jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board2,,,
Loading

0 comments on commit 828f498

Please sign in to comment.