From aa07ad9dd6f3f9c92042743b813400fc4deb36c3 Mon Sep 17 00:00:00 2001 From: Lynwee Hou Date: Wed, 22 Nov 2023 14:11:10 +0800 Subject: [PATCH] feat(db): add date relation fields to devops tables --- .../domainlayer/devops/cicd_deployment.go | 17 ++-- .../devops/cicd_deployment_commit.go | 38 ++++---- .../domainlayer/devops/cicd_pipeline.go | 21 ++--- .../models/domainlayer/devops/cicd_task.go | 21 +++-- .../core/models/domainlayer/devops/common.go | 27 ++++++ ...22_add_some_date_field_in_devops_tables.go | 88 +++++++++++++++++++ .../core/models/migrationscripts/register.go | 1 + .../bamboo/tasks/deploy_build_convertor.go | 14 +-- .../bamboo/tasks/job_build_convertor.go | 13 +-- .../bamboo/tasks/plan_build_convertor.go | 11 +-- .../bitbucket/tasks/deployment_convertor.go | 25 +++--- .../bitbucket/tasks/pipeline_convertor.go | 14 +-- .../tasks/pipeline_steps_convertor.go | 2 +- .../plugins/circleci/tasks/job_converter.go | 18 ++-- .../circleci/tasks/workflow_converter.go | 12 +-- .../tasks/deployment_commits_generator.go | 16 ++-- .../github/tasks/cicd_job_convertor.go | 16 ++-- .../github/tasks/cicd_run_convertor.go | 17 ++-- .../github/tasks/deployment_convertor.go | 18 ++-- .../gitlab/tasks/deployment_convertor.go | 21 ++--- backend/plugins/gitlab/tasks/job_convertor.go | 13 +-- .../gitlab/tasks/pipeline_convertor.go | 12 +-- .../jenkins/tasks/build_cicd_convertor.go | 42 +++++---- .../plugins/jenkins/tasks/stage_convertor.go | 22 ++--- backend/plugins/webhook/api/deployments.go | 18 ++-- 25 files changed, 327 insertions(+), 190 deletions(-) create mode 100644 backend/core/models/domainlayer/devops/common.go create mode 100644 backend/core/models/migrationscripts/20231122_add_some_date_field_in_devops_tables.go diff --git a/backend/core/models/domainlayer/devops/cicd_deployment.go b/backend/core/models/domainlayer/devops/cicd_deployment.go index fb63d1dba9f..ca24045e0fa 100644 --- a/backend/core/models/domainlayer/devops/cicd_deployment.go +++ b/backend/core/models/domainlayer/devops/cicd_deployment.go @@ -19,20 +19,17 @@ package devops import ( "github.com/apache/incubator-devlake/core/models/domainlayer" - "time" ) type CICDDeployment struct { domainlayer.DomainEntity - CicdScopeId string `gorm:"index;type:varchar(255)"` - Name string `gorm:"type:varchar(255)"` - Result string `gorm:"type:varchar(100)"` - Status string `gorm:"type:varchar(100)"` - Environment string `gorm:"type:varchar(255)"` - CreatedDate time.Time - StartedDate *time.Time - FinishedDate *time.Time - DurationSec *float64 + CicdScopeId string `gorm:"index;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Result string `gorm:"type:varchar(100)"` + Status string `gorm:"type:varchar(100)"` + Environment string `gorm:"type:varchar(255)"` + ItemDateInfo + DurationSec *float64 } func (CICDDeployment) TableName() string { diff --git a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go index 5d37448ad23..18bc468936c 100644 --- a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go +++ b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go @@ -18,22 +18,18 @@ limitations under the License. package devops import ( - "time" - "github.com/apache/incubator-devlake/core/models/domainlayer" ) type CicdDeploymentCommit struct { domainlayer.DomainEntity - CicdScopeId string `gorm:"index;type:varchar(255)"` - CicdDeploymentId string `gorm:"type:varchar(255)"` // if it is converted from a cicd_pipeline_commit - Name string `gorm:"type:varchar(255)"` - Result string `gorm:"type:varchar(100)"` - Status string `gorm:"type:varchar(100)"` - Environment string `gorm:"type:varchar(255)"` - CreatedDate time.Time - StartedDate *time.Time - FinishedDate *time.Time + CicdScopeId string `gorm:"index;type:varchar(255)"` + CicdDeploymentId string `gorm:"type:varchar(255)"` // if it is converted from a cicd_pipeline_commit + Name string `gorm:"type:varchar(255)"` + Result string `gorm:"type:varchar(100)"` + Status string `gorm:"type:varchar(100)"` + Environment string `gorm:"type:varchar(255)"` + ItemDateInfo DurationSec *float64 CommitSha string `gorm:"primaryKey;type:varchar(255)"` RefName string `gorm:"type:varchar(255)"` // to delete? @@ -52,14 +48,16 @@ func (cicdDeploymentCommit CicdDeploymentCommit) ToDeployment() *CICDDeployment Id: cicdDeploymentCommit.CicdDeploymentId, NoPKModel: cicdDeploymentCommit.DomainEntity.NoPKModel, }, - CicdScopeId: cicdDeploymentCommit.CicdScopeId, - Name: cicdDeploymentCommit.Name, - Result: cicdDeploymentCommit.Result, - Status: cicdDeploymentCommit.Status, - Environment: cicdDeploymentCommit.Environment, - CreatedDate: cicdDeploymentCommit.CreatedDate, - StartedDate: cicdDeploymentCommit.StartedDate, - FinishedDate: cicdDeploymentCommit.FinishedDate, - DurationSec: cicdDeploymentCommit.DurationSec, + CicdScopeId: cicdDeploymentCommit.CicdScopeId, + Name: cicdDeploymentCommit.Name, + Result: cicdDeploymentCommit.Result, + Status: cicdDeploymentCommit.Status, + Environment: cicdDeploymentCommit.Environment, + ItemDateInfo: ItemDateInfo{ + CreatedDate: cicdDeploymentCommit.CreatedDate, + StartedDate: cicdDeploymentCommit.StartedDate, + FinishedDate: cicdDeploymentCommit.FinishedDate, + }, + DurationSec: cicdDeploymentCommit.DurationSec, } } diff --git a/backend/core/models/domainlayer/devops/cicd_pipeline.go b/backend/core/models/domainlayer/devops/cicd_pipeline.go index 5bc4f3250f8..605764ba768 100644 --- a/backend/core/models/domainlayer/devops/cicd_pipeline.go +++ b/backend/core/models/domainlayer/devops/cicd_pipeline.go @@ -18,25 +18,22 @@ limitations under the License. package devops import ( - "strings" - "time" - "github.com/spf13/cast" + "strings" "github.com/apache/incubator-devlake/core/models/domainlayer" ) type CICDPipeline struct { domainlayer.DomainEntity - Name string `gorm:"type:varchar(255)"` - Result string `gorm:"type:varchar(100)"` - Status string `gorm:"type:varchar(100)"` - Type string `gorm:"type:varchar(100);comment: to indicate this is CI or CD"` - DurationSec float64 - Environment string `gorm:"type:varchar(255)"` - CreatedDate time.Time - FinishedDate *time.Time - CicdScopeId string `gorm:"index;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Result string `gorm:"type:varchar(100)"` + Status string `gorm:"type:varchar(100)"` + Type string `gorm:"type:varchar(100);comment: to indicate this is CI or CD"` + DurationSec float64 + Environment string `gorm:"type:varchar(255)"` + ItemDateInfo + CicdScopeId string `gorm:"index;type:varchar(255)"` } func (CICDPipeline) TableName() string { diff --git a/backend/core/models/domainlayer/devops/cicd_task.go b/backend/core/models/domainlayer/devops/cicd_task.go index 8314d4ccebb..e3a4aa9a356 100644 --- a/backend/core/models/domainlayer/devops/cicd_task.go +++ b/backend/core/models/domainlayer/devops/cicd_task.go @@ -19,7 +19,6 @@ package devops import ( "github.com/apache/incubator-devlake/core/models/domainlayer" - "time" ) const ( @@ -39,16 +38,16 @@ const ENV_NAME_PATTERN = "ENV_NAME_PATTERN" type CICDTask struct { domainlayer.DomainEntity - Name string `gorm:"type:varchar(255)"` - PipelineId string `gorm:"index;type:varchar(255)"` - Result string `gorm:"type:varchar(100)"` - Status string `gorm:"type:varchar(100)"` - Type string `gorm:"type:varchar(100);comment: to indicate this is CI or CD"` - Environment string `gorm:"type:varchar(255)"` - DurationSec float64 - StartedDate time.Time - FinishedDate *time.Time - CicdScopeId string `gorm:"index;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + PipelineId string `gorm:"index;type:varchar(255)"` + Result string `gorm:"type:varchar(100)"` + Status string `gorm:"type:varchar(100)"` + Type string `gorm:"type:varchar(100);comment: to indicate this is CI or CD"` + Environment string `gorm:"type:varchar(255)"` + DurationSec float64 + ItemDateInfo + //StartedDate time.Time // notice here + CicdScopeId string `gorm:"index;type:varchar(255)"` } func (CICDTask) TableName() string { diff --git a/backend/core/models/domainlayer/devops/common.go b/backend/core/models/domainlayer/devops/common.go new file mode 100644 index 00000000000..d2aded1cb7b --- /dev/null +++ b/backend/core/models/domainlayer/devops/common.go @@ -0,0 +1,27 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package devops + +import "time" + +type ItemDateInfo struct { + CreatedDate time.Time + StartedDate *time.Time + QueuedDate *time.Time + FinishedDate *time.Time +} diff --git a/backend/core/models/migrationscripts/20231122_add_some_date_field_in_devops_tables.go b/backend/core/models/migrationscripts/20231122_add_some_date_field_in_devops_tables.go new file mode 100644 index 00000000000..d4abac50556 --- /dev/null +++ b/backend/core/models/migrationscripts/20231122_add_some_date_field_in_devops_tables.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "time" +) + +var _ plugin.MigrationScript = (*addSomeDateFieldsToDevopsTables)(nil) + +type cicdDeployment20231122 struct { + QueuedDate *time.Time +} + +func (cicdDeployment20231122) TableName() string { + return "cicd_deployments" +} + +type cicdDeploymentCommit20231122 struct { + QueuedDate *time.Time +} + +func (cicdDeploymentCommit20231122) TableName() string { + return "cicd_deployment_commits" +} + +type cicdPipeline20231122 struct { + StartedDate *time.Time + QueuedDate *time.Time +} + +func (cicdPipeline20231122) TableName() string { + return "cicd_pipelines" +} + +type cicdTask20231122 struct { + CreatedDate time.Time + QueuedDate *time.Time +} + +func (cicdTask20231122) TableName() string { + return "cicd_tasks" +} + +type addSomeDateFieldsToDevopsTables struct{} + +func (u *addSomeDateFieldsToDevopsTables) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.AutoMigrate(&cicdPipeline20231122{}); err != nil { + return err + } + if err := db.AutoMigrate(&cicdTask20231122{}); err != nil { + return err + } + if err := db.AutoMigrate(&cicdDeployment20231122{}); err != nil { + return err + } + if err := db.AutoMigrate(&cicdDeploymentCommit20231122{}); err != nil { + return err + } + return nil +} + +func (*addSomeDateFieldsToDevopsTables) Version() uint64 { + return 20231122140000 +} + +func (*addSomeDateFieldsToDevopsTables) Name() string { + return "change duration_sec field to float64 in all related tables" +} diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go index 241b6af1976..590ad913c4c 100644 --- a/backend/core/models/migrationscripts/register.go +++ b/backend/core/models/migrationscripts/register.go @@ -97,5 +97,6 @@ func All() []plugin.MigrationScript { new(addIssueCustomArrayField), new(removePositionFromPullRequestComments), new(changeDurationSecToFloat64), + new(addSomeDateFieldsToDevopsTables), } } diff --git a/backend/plugins/bamboo/tasks/deploy_build_convertor.go b/backend/plugins/bamboo/tasks/deploy_build_convertor.go index 40b8adad04f..740ea28a352 100644 --- a/backend/plugins/bamboo/tasks/deploy_build_convertor.go +++ b/backend/plugins/bamboo/tasks/deploy_build_convertor.go @@ -122,12 +122,14 @@ func ConvertDeployBuilds(taskCtx plugin.SubTaskContext) errors.Error { InProgress: []string{StatusInProgress, StatusPending, StatusQueued}, Default: devops.STATUS_OTHER, }, input.LifeCycleState), - Environment: input.Environment, - StartedDate: input.StartedDate, - FinishedDate: input.FinishedDate, - CommitSha: input.VcsRevisionKey, - RefName: input.PlanBranchName, - RepoId: strconv.Itoa(input.RepositoryId), + Environment: input.Environment, + ItemDateInfo: devops.ItemDateInfo{ + StartedDate: input.StartedDate, + FinishedDate: input.FinishedDate, + }, + CommitSha: input.VcsRevisionKey, + RefName: input.PlanBranchName, + RepoId: strconv.Itoa(input.RepositoryId), } deploymentCommit.CreatedDate = time.Now() if input.StartedDate != nil { diff --git a/backend/plugins/bamboo/tasks/job_build_convertor.go b/backend/plugins/bamboo/tasks/job_build_convertor.go index 89784a5d7f9..daac375e8fb 100644 --- a/backend/plugins/bamboo/tasks/job_build_convertor.go +++ b/backend/plugins/bamboo/tasks/job_build_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -28,6 +26,7 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/bamboo/models" + "reflect" ) var ConvertJobBuildsMeta = plugin.SubTaskMeta{ @@ -66,10 +65,12 @@ func ConvertJobBuilds(taskCtx plugin.SubTaskContext) errors.Error { DomainEntity: domainlayer.DomainEntity{Id: jobBuildIdGen.Generate(data.Options.ConnectionId, line.JobBuildKey)}, Name: line.JobName, DurationSec: float64(line.BuildDurationInSeconds), - StartedDate: *line.BuildStartedTime, - FinishedDate: line.BuildCompletedDate, - PipelineId: planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey), - CicdScopeId: planIdGen.Generate(data.Options.ConnectionId, data.Options.PlanKey), + ItemDateInfo: devops.ItemDateInfo{ + StartedDate: line.BuildStartedTime, + FinishedDate: line.BuildCompletedDate, + }, + PipelineId: planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey), + CicdScopeId: planIdGen.Generate(data.Options.ConnectionId, data.Options.PlanKey), Result: devops.GetResult(&devops.ResultRule{ Success: []string{ResultSuccess, ResultSuccessful}, diff --git a/backend/plugins/bamboo/tasks/plan_build_convertor.go b/backend/plugins/bamboo/tasks/plan_build_convertor.go index 40e797d5c80..8d473983262 100644 --- a/backend/plugins/bamboo/tasks/plan_build_convertor.go +++ b/backend/plugins/bamboo/tasks/plan_build_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -28,6 +26,7 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/bamboo/models" + "reflect" ) var ConvertPlanBuildsMeta = plugin.SubTaskMeta{ @@ -65,9 +64,11 @@ func ConvertPlanBuilds(taskCtx plugin.SubTaskContext) errors.Error { DomainEntity: domainlayer.DomainEntity{Id: planBuildIdGen.Generate(data.Options.ConnectionId, line.PlanBuildKey)}, Name: line.GenerateCICDPipeLineName(), DurationSec: float64(line.BuildDurationInSeconds), - CreatedDate: *line.BuildStartedTime, - FinishedDate: line.BuildCompletedDate, - CicdScopeId: planIdGen.Generate(data.Options.ConnectionId, data.Options.PlanKey), + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: *line.BuildStartedTime, + FinishedDate: line.BuildCompletedDate, + }, + CicdScopeId: planIdGen.Generate(data.Options.ConnectionId, data.Options.PlanKey), Result: devops.GetResult(&devops.ResultRule{ Success: []string{ResultSuccess, ResultSuccessful}, diff --git a/backend/plugins/bitbucket/tasks/deployment_convertor.go b/backend/plugins/bitbucket/tasks/deployment_convertor.go index b19512d93e3..4441f09843b 100644 --- a/backend/plugins/bitbucket/tasks/deployment_convertor.go +++ b/backend/plugins/bitbucket/tasks/deployment_convertor.go @@ -18,9 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "strings" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -29,6 +26,8 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/bitbucket/models" + "reflect" + "strings" ) var ConvertiDeploymentMeta = plugin.SubTaskMeta{ @@ -96,15 +95,17 @@ func ConvertDeployments(taskCtx plugin.SubTaskContext) errors.Error { InProgress: []string{models.IN_PROGRESS}, Default: devops.STATUS_OTHER, }, bitbucketDeployment.Status), - Environment: strings.ToUpper(bitbucketDeployment.Environment), // or bitbucketDeployment.EnvironmentType, they are same so far. - CreatedDate: *bitbucketDeployment.CreatedOn, - StartedDate: bitbucketDeployment.StartedOn, - FinishedDate: bitbucketDeployment.CompletedOn, - DurationSec: duration, - CommitSha: bitbucketDeployment.CommitSha, - RefName: bitbucketDeployment.RefName, - RepoId: repoId, - RepoUrl: repo.HTMLUrl, + Environment: strings.ToUpper(bitbucketDeployment.Environment), // or bitbucketDeployment.EnvironmentType, they are same so far. + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: *bitbucketDeployment.CreatedOn, + StartedDate: bitbucketDeployment.StartedOn, + FinishedDate: bitbucketDeployment.CompletedOn, + }, + DurationSec: duration, + CommitSha: bitbucketDeployment.CommitSha, + RefName: bitbucketDeployment.RefName, + RepoId: repoId, + RepoUrl: repo.HTMLUrl, } if domainDeployCommit.Environment == devops.TEST { // Theoretically, environment cannot be "Test" according to diff --git a/backend/plugins/bitbucket/tasks/pipeline_convertor.go b/backend/plugins/bitbucket/tasks/pipeline_convertor.go index 7132e753be8..18c6bb1ac37 100644 --- a/backend/plugins/bitbucket/tasks/pipeline_convertor.go +++ b/backend/plugins/bitbucket/tasks/pipeline_convertor.go @@ -97,12 +97,14 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error { InProgress: []string{models.IN_PROGRESS, models.PENDING, models.RUNNING, models.PAUSED, models.BUILDING}, Default: devops.STATUS_OTHER, }, bitbucketPipeline.Status), - Type: bitbucketPipeline.Type, - Environment: bitbucketPipeline.Environment, - CreatedDate: createdAt, - DurationSec: float64(bitbucketPipeline.DurationInSeconds), - FinishedDate: bitbucketPipeline.BitbucketCompleteOn, - CicdScopeId: repoId, + Type: bitbucketPipeline.Type, + Environment: bitbucketPipeline.Environment, + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: createdAt, + FinishedDate: bitbucketPipeline.BitbucketCompleteOn, + }, + DurationSec: float64(bitbucketPipeline.DurationInSeconds), + CicdScopeId: repoId, } results = append(results, domainPipelineCommit, domainPipeline) return results, nil diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go index 0eb10bb00fd..681a2b085a9 100644 --- a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go +++ b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go @@ -84,7 +84,7 @@ func ConvertPipelineSteps(taskCtx plugin.SubTaskContext) errors.Error { if bitbucketPipelineStep.StartedOn == nil { return nil, nil } - domainTask.StartedDate = *bitbucketPipelineStep.StartedOn + domainTask.StartedDate = bitbucketPipelineStep.StartedOn // rebuild the FinishedDate if domainTask.Status == devops.STATUS_DONE { domainTask.FinishedDate = bitbucketPipelineStep.CompletedOn diff --git a/backend/plugins/circleci/tasks/job_converter.go b/backend/plugins/circleci/tasks/job_converter.go index 9fbfc40f17d..5bed6acef00 100644 --- a/backend/plugins/circleci/tasks/job_converter.go +++ b/backend/plugins/circleci/tasks/job_converter.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -27,6 +25,7 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/circleci/models" + "reflect" ) var ConvertJobsMeta = plugin.SubTaskMeta{ @@ -56,16 +55,19 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) errors.Error { Input: cursor, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { userTool := inputRow.(*models.CircleciJob) + startedAt := userTool.StartedAt.ToTime() task := &devops.CICDTask{ DomainEntity: domainlayer.DomainEntity{ Id: getJobIdGen().Generate(data.Options.ConnectionId, userTool.WorkflowId, userTool.Id), }, - CicdScopeId: getProjectIdGen().Generate(data.Options.ConnectionId, data.Project.Id), - Name: userTool.Name, - PipelineId: userTool.PipelineId, - StartedDate: userTool.StartedAt.ToTime(), - FinishedDate: userTool.StoppedAt.ToNullableTime(), - DurationSec: userTool.DurationSec, + CicdScopeId: getProjectIdGen().Generate(data.Options.ConnectionId, data.Project.Id), + Name: userTool.Name, + PipelineId: userTool.PipelineId, + ItemDateInfo: devops.ItemDateInfo{ + StartedDate: &startedAt, + FinishedDate: userTool.StoppedAt.ToNullableTime(), + }, + DurationSec: userTool.DurationSec, // reference: https://circleci.com/docs/api/v2/index.html#operation/getJobDetails Status: devops.GetStatus(&devops.StatusRule{ Done: []string{"canceled", "failed", "failing", "success", "not_run", "error", "infrastructure_fail", "timedout", "terminated-unknown"}, // on_hold,blocked diff --git a/backend/plugins/circleci/tasks/workflow_converter.go b/backend/plugins/circleci/tasks/workflow_converter.go index b7c6c2eca4f..8db9092286f 100644 --- a/backend/plugins/circleci/tasks/workflow_converter.go +++ b/backend/plugins/circleci/tasks/workflow_converter.go @@ -60,11 +60,13 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error { DomainEntity: domainlayer.DomainEntity{ Id: getPipelineIdGen().Generate(data.Options.ConnectionId, userTool.Id), }, - Name: userTool.Name, - DurationSec: userTool.DurationSec, - CreatedDate: userTool.CreatedAt.ToTime(), - FinishedDate: userTool.StoppedAt.ToNullableTime(), - CicdScopeId: getProjectIdGen().Generate(data.Options.ConnectionId, userTool.ProjectSlug), + Name: userTool.Name, + DurationSec: userTool.DurationSec, + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: userTool.CreatedAt.ToTime(), + FinishedDate: userTool.StoppedAt.ToNullableTime(), + }, + CicdScopeId: getProjectIdGen().Generate(data.Options.ConnectionId, userTool.ProjectSlug), // reference: https://circleci.com/docs/api/v2/index.html#operation/getWorkflowById Status: devops.GetStatus(&devops.StatusRule{ Done: []string{"canceled", "failed", "failing", "success", "not_run", "error"}, // on_hold diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go index 7061313b3fb..2c12dfece4a 100644 --- a/backend/plugins/dora/tasks/deployment_commits_generator.go +++ b/backend/plugins/dora/tasks/deployment_commits_generator.go @@ -127,13 +127,15 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error { Result: pipelineCommit.Result, Status: pipelineCommit.Status, Environment: pipelineCommit.Environment, - CreatedDate: *pipelineCommit.CreatedDate, - FinishedDate: pipelineCommit.FinishedDate, - DurationSec: pipelineCommit.DurationSec, - CommitSha: pipelineCommit.CommitSha, - RefName: pipelineCommit.Branch, - RepoId: pipelineCommit.RepoId, - RepoUrl: pipelineCommit.RepoUrl, + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: *pipelineCommit.CreatedDate, + FinishedDate: pipelineCommit.FinishedDate, + }, + DurationSec: pipelineCommit.DurationSec, + CommitSha: pipelineCommit.CommitSha, + RefName: pipelineCommit.Branch, + RepoId: pipelineCommit.RepoId, + RepoUrl: pipelineCommit.RepoUrl, } if pipelineCommit.FinishedDate != nil && pipelineCommit.DurationSec != nil { s := pipelineCommit.FinishedDate.Add(-time.Duration(*pipelineCommit.DurationSec) * time.Second) diff --git a/backend/plugins/github/tasks/cicd_job_convertor.go b/backend/plugins/github/tasks/cicd_job_convertor.go index 7be00b0c7a3..12b42a67da9 100644 --- a/backend/plugins/github/tasks/cicd_job_convertor.go +++ b/backend/plugins/github/tasks/cicd_job_convertor.go @@ -89,13 +89,15 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) { domainJob := &devops.CICDTask{ DomainEntity: domainlayer.DomainEntity{Id: jobIdGen.Generate(data.Options.ConnectionId, line.RunID, line.ID)}, - Name: line.Name, - StartedDate: *line.StartedAt, - FinishedDate: line.CompletedAt, - PipelineId: runIdGen.Generate(data.Options.ConnectionId, line.RepoId, line.RunID), - CicdScopeId: repoIdGen.Generate(data.Options.ConnectionId, line.RepoId), - Type: line.Type, - Environment: line.Environment, + Name: line.Name, + ItemDateInfo: devops.ItemDateInfo{ + StartedDate: line.StartedAt, + FinishedDate: line.CompletedAt, + }, + PipelineId: runIdGen.Generate(data.Options.ConnectionId, line.RepoId, line.RunID), + CicdScopeId: repoIdGen.Generate(data.Options.ConnectionId, line.RepoId), + Type: line.Type, + Environment: line.Environment, Result: devops.GetResult(&devops.ResultRule{ Success: []string{StatusSuccess}, Failure: []string{StatusFailure, StatusCancelled, StatusTimedOut, StatusStartUpFailure}, diff --git a/backend/plugins/github/tasks/cicd_run_convertor.go b/backend/plugins/github/tasks/cicd_run_convertor.go index a6b08e5eb79..131a3141ad9 100644 --- a/backend/plugins/github/tasks/cicd_run_convertor.go +++ b/backend/plugins/github/tasks/cicd_run_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -28,6 +26,7 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" + "reflect" ) func init() { @@ -91,12 +90,14 @@ func ConvertRuns(taskCtx plugin.SubTaskContext) errors.Error { DomainEntity: domainlayer.DomainEntity{Id: runIdGen.Generate( data.Options.ConnectionId, line.RepoId, line.ID), }, - Name: line.Name, - CreatedDate: *line.GithubCreatedAt, - FinishedDate: line.GithubUpdatedAt, - CicdScopeId: repoIdGen.Generate(data.Options.ConnectionId, line.RepoId), - Type: line.Type, - Environment: line.Environment, + Name: line.Name, + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: *line.GithubCreatedAt, + FinishedDate: line.GithubUpdatedAt, + }, + CicdScopeId: repoIdGen.Generate(data.Options.ConnectionId, line.RepoId), + Type: line.Type, + Environment: line.Environment, Result: devops.GetResult(&devops.ResultRule{ Success: []string{StatusSuccess}, Failure: []string{StatusFailure, StatusCancelled, StatusTimedOut, StatusStartUpFailure}, diff --git a/backend/plugins/github/tasks/deployment_convertor.go b/backend/plugins/github/tasks/deployment_convertor.go index 7047b686272..91351fefdf8 100644 --- a/backend/plugins/github/tasks/deployment_convertor.go +++ b/backend/plugins/github/tasks/deployment_convertor.go @@ -85,14 +85,16 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error { InProgress: []string{StatusInProgress, StatusQueued, StatusWaiting, StatusPending}, Default: devops.STATUS_OTHER, }, githubDeployment.State), - Environment: githubDeployment.Environment, - CreatedDate: githubDeployment.CreatedDate, - StartedDate: &githubDeployment.CreatedDate, // fixme there is no such field - FinishedDate: &githubDeployment.UpdatedDate, // fixme there is no such field - CommitSha: githubDeployment.CommitOid, - RefName: githubDeployment.RefName, - RepoId: deploymentScopeIdGen.Generate(githubDeployment.ConnectionId, githubDeployment.GithubId), - RepoUrl: githubDeployment.RepositoryUrl, + Environment: githubDeployment.Environment, + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: githubDeployment.CreatedDate, + StartedDate: &githubDeployment.CreatedDate, // fixme there is no such field + FinishedDate: &githubDeployment.UpdatedDate, // fixme there is no such field + }, + CommitSha: githubDeployment.CommitOid, + RefName: githubDeployment.RefName, + RepoId: deploymentScopeIdGen.Generate(githubDeployment.ConnectionId, githubDeployment.GithubId), + RepoUrl: githubDeployment.RepositoryUrl, } durationSec := githubDeployment.UpdatedDate.Sub(githubDeployment.CreatedDate).Seconds() diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go b/backend/plugins/gitlab/tasks/deployment_convertor.go index 81d9b0b613c..6d3988a4678 100644 --- a/backend/plugins/gitlab/tasks/deployment_convertor.go +++ b/backend/plugins/gitlab/tasks/deployment_convertor.go @@ -19,8 +19,6 @@ package tasks import ( "fmt" - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -30,6 +28,7 @@ import ( "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" "github.com/spf13/cast" + "reflect" ) var _ plugin.SubTaskEntryPoint = ConvertDeployment @@ -103,14 +102,16 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error { InProgress: []string{StatusRunning}, Default: devops.STATUS_OTHER, }, gitlabDeployment.Status), - Environment: gitlabDeployment.Environment, - CreatedDate: gitlabDeployment.CreatedDate, - StartedDate: gitlabDeployment.DeployableStartedAt, - FinishedDate: gitlabDeployment.DeployableFinishedAt, - CommitSha: gitlabDeployment.Sha, - RefName: gitlabDeployment.Ref, - RepoId: projectIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId), - RepoUrl: repo.WebUrl, + Environment: gitlabDeployment.Environment, + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: gitlabDeployment.CreatedDate, + StartedDate: gitlabDeployment.DeployableStartedAt, + FinishedDate: gitlabDeployment.DeployableFinishedAt, + }, + CommitSha: gitlabDeployment.Sha, + RefName: gitlabDeployment.Ref, + RepoId: projectIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId), + RepoUrl: repo.WebUrl, } if duration != nil { domainDeployCommit.DurationSec = duration diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go index 3cd02d3b964..10f94c5b3e4 100644 --- a/backend/plugins/gitlab/tasks/job_convertor.go +++ b/backend/plugins/gitlab/tasks/job_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -28,6 +26,7 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models" + "reflect" ) func init() { @@ -95,10 +94,12 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) { Default: devops.STATUS_OTHER, }, gitlabJob.Status), - DurationSec: gitlabJob.Duration, - StartedDate: *startedAt, - FinishedDate: gitlabJob.FinishedAt, - CicdScopeId: projectIdGen.Generate(data.Options.ConnectionId, gitlabJob.ProjectId), + DurationSec: gitlabJob.Duration, + ItemDateInfo: devops.ItemDateInfo{ + StartedDate: startedAt, + FinishedDate: gitlabJob.FinishedAt, + }, + CicdScopeId: projectIdGen.Generate(data.Options.ConnectionId, gitlabJob.ProjectId), } domainJob.Type = regexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, gitlabJob.Name) domainJob.Environment = regexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, gitlabJob.Name) diff --git a/backend/plugins/gitlab/tasks/pipeline_convertor.go b/backend/plugins/gitlab/tasks/pipeline_convertor.go index 4192c6a0264..a5fe70ea56d 100644 --- a/backend/plugins/gitlab/tasks/pipeline_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_convertor.go @@ -95,11 +95,13 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error { InProgress: []string{StatusRunning, StatusWaitingForResource, StatusPending, StatusPreparing}, Default: devops.STATUS_OTHER, }, gitlabPipeline.Status), - CreatedDate: startedAt, - FinishedDate: gitlabPipeline.GitlabUpdatedAt, - CicdScopeId: projectIdGen.Generate(data.Options.ConnectionId, gitlabPipeline.ProjectId), - Environment: gitlabPipeline.Environment, - Type: gitlabPipeline.Type, + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: startedAt, + FinishedDate: gitlabPipeline.GitlabUpdatedAt, + }, + CicdScopeId: projectIdGen.Generate(data.Options.ConnectionId, gitlabPipeline.ProjectId), + Environment: gitlabPipeline.Environment, + Type: gitlabPipeline.Type, } // rebuild the FinishedDate and DurationSec by Status diff --git a/backend/plugins/jenkins/tasks/build_cicd_convertor.go b/backend/plugins/jenkins/tasks/build_cicd_convertor.go index 485814d3e79..ce36530c026 100644 --- a/backend/plugins/jenkins/tasks/build_cicd_convertor.go +++ b/backend/plugins/jenkins/tasks/build_cicd_convertor.go @@ -102,15 +102,17 @@ func ConvertBuildsToCicdTasks(taskCtx plugin.SubTaskContext) (err errors.Error) DomainEntity: domainlayer.DomainEntity{ Id: buildIdGen.Generate(jenkinsBuild.ConnectionId, jenkinsBuild.FullName), }, - Name: jenkinsBuild.FullName, - Result: jenkinsPipelineResult, - Status: jenkinsPipelineStatus, - FinishedDate: jenkinsPipelineFinishedDate, - DurationSec: durationSec, - CreatedDate: jenkinsBuild.StartTime, - CicdScopeId: jobIdGen.Generate(jenkinsBuild.ConnectionId, data.Options.JobFullName), - Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, jenkinsBuild.FullName), - Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, jenkinsBuild.FullName), + Name: jenkinsBuild.FullName, + Result: jenkinsPipelineResult, + Status: jenkinsPipelineStatus, + ItemDateInfo: devops.ItemDateInfo{ + FinishedDate: jenkinsPipelineFinishedDate, + CreatedDate: jenkinsBuild.StartTime, + }, + DurationSec: durationSec, + CicdScopeId: jobIdGen.Generate(jenkinsBuild.ConnectionId, data.Options.JobFullName), + Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, jenkinsBuild.FullName), + Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, jenkinsBuild.FullName), } jenkinsPipeline.RawDataOrigin = jenkinsBuild.RawDataOrigin results = append(results, jenkinsPipeline) @@ -120,16 +122,18 @@ func ConvertBuildsToCicdTasks(taskCtx plugin.SubTaskContext) (err errors.Error) DomainEntity: domainlayer.DomainEntity{ Id: buildIdGen.Generate(jenkinsBuild.ConnectionId, jenkinsBuild.FullName), }, - Name: data.Options.JobFullName, - Result: jenkinsPipelineResult, - Status: jenkinsPipelineStatus, - DurationSec: durationSec, - StartedDate: jenkinsBuild.StartTime, - FinishedDate: jenkinsPipelineFinishedDate, - CicdScopeId: jobIdGen.Generate(jenkinsBuild.ConnectionId, data.Options.JobFullName), - Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, jenkinsBuild.FullName), - Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, jenkinsBuild.FullName), - PipelineId: buildIdGen.Generate(jenkinsBuild.ConnectionId, jenkinsBuild.FullName), + Name: data.Options.JobFullName, + Result: jenkinsPipelineResult, + Status: jenkinsPipelineStatus, + DurationSec: durationSec, + ItemDateInfo: devops.ItemDateInfo{ + StartedDate: &jenkinsBuild.StartTime, + FinishedDate: jenkinsPipelineFinishedDate, + }, + CicdScopeId: jobIdGen.Generate(jenkinsBuild.ConnectionId, data.Options.JobFullName), + Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, jenkinsBuild.FullName), + Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, jenkinsBuild.FullName), + PipelineId: buildIdGen.Generate(jenkinsBuild.ConnectionId, jenkinsBuild.FullName), } results = append(results, jenkinsTask) diff --git a/backend/plugins/jenkins/tasks/stage_convertor.go b/backend/plugins/jenkins/tasks/stage_convertor.go index bb439e7b3d9..da3b7686638 100644 --- a/backend/plugins/jenkins/tasks/stage_convertor.go +++ b/backend/plugins/jenkins/tasks/stage_convertor.go @@ -127,16 +127,18 @@ func ConvertStages(taskCtx plugin.SubTaskContext) (err errors.Error) { DomainEntity: domainlayer.DomainEntity{ Id: stageIdGen.Generate(body.ConnectionId, body.BuildName, body.ID), }, - Name: body.Name, - PipelineId: buildIdGen.Generate(body.ConnectionId, body.BuildName), - Result: jenkinsTaskResult, - Status: jenkinsTaskStatus, - DurationSec: durationSec, - StartedDate: startedDate, - FinishedDate: jenkinsTaskFinishedDate, - CicdScopeId: jobIdGen.Generate(body.ConnectionId, data.Options.JobFullName), - Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, body.Name), - Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, body.Name), + Name: body.Name, + PipelineId: buildIdGen.Generate(body.ConnectionId, body.BuildName), + Result: jenkinsTaskResult, + Status: jenkinsTaskStatus, + DurationSec: durationSec, + ItemDateInfo: devops.ItemDateInfo{ + StartedDate: &startedDate, + FinishedDate: jenkinsTaskFinishedDate, + }, + CicdScopeId: jobIdGen.Generate(body.ConnectionId, data.Options.JobFullName), + Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, body.Name), + Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, body.Name), } results = append(results, jenkinsTask) return results, nil diff --git a/backend/plugins/webhook/api/deployments.go b/backend/plugins/webhook/api/deployments.go index bb60ffcd39e..68d44568ab3 100644 --- a/backend/plugins/webhook/api/deployments.go +++ b/backend/plugins/webhook/api/deployments.go @@ -117,14 +117,16 @@ func PostDeploymentCicdTask(input *plugin.ApiResourceInput) (*plugin.ApiResource Result: request.Result, Status: devops.STATUS_DONE, Environment: request.Environment, - CreatedDate: *request.CreatedDate, - StartedDate: request.StartedDate, - FinishedDate: request.FinishedDate, - DurationSec: &duration, - CommitSha: request.CommitSha, - RefName: request.RefName, - RepoId: request.RepoId, - RepoUrl: request.RepoUrl, + ItemDateInfo: devops.ItemDateInfo{ + CreatedDate: *request.CreatedDate, + StartedDate: request.StartedDate, + FinishedDate: request.FinishedDate, + }, + DurationSec: &duration, + CommitSha: request.CommitSha, + RefName: request.RefName, + RepoId: request.RepoId, + RepoUrl: request.RepoUrl, } err = tx.CreateOrUpdate(deploymentCommit) if err != nil {