Merge branch 'main' into fix-6139

backend/core/models/locking.go

@@ -43,7 +43,7 @@ func (LockingHistory) TableName() string {
 
 // LockingStub does nothing but offer a locking target
 type LockingStub struct {
-	Stub string
+	Stub string `gorm:"primaryKey"`
 }
 
 func (LockingStub) TableName() string {

backend/core/runner/run_pipeline.go

@@ -68,6 +68,11 @@ func runPipelineTasks(
 		return err
 	}
 
+	// if pipeline has been cancelled, just return.
+	if dbPipeline.Status == models.TASK_CANCELLED {
+		return nil
+	}
+
 	// This double for loop executes each set of tasks sequentially while
 	// executing the set of tasks concurrently.
 	for i, row := range taskIds {
@@ -90,6 +95,10 @@ func runPipelineTasks(
 			}
 		}
 	}
-	log.Info("pipeline finished in %d ms: %v", time.Now().UnixMilli()-dbPipeline.BeganAt.UnixMilli(), err)
+	if dbPipeline.BeganAt != nil {
+		log.Info("pipeline finished in %d ms: %v", time.Now().UnixMilli()-dbPipeline.BeganAt.UnixMilli(), err)
+	} else {
+		log.Info("pipeline finished at %d ms: %v", time.Now().UnixMilli(), err)
+	}
 	return err
 }

backend/core/runner/run_task.go

@@ -20,6 +20,7 @@ package runner
 import (
 	gocontext "context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/apache/incubator-devlake/core/context"
@@ -214,8 +215,12 @@ func RunPluginSubTasks(
 		}
 	}
 
-	// make sure `Required` subtasks are always enabled
+	// 1. make sure `Collect` subtasks skip if `SkipCollectors` is true
+	// 2. make sure `Required` subtasks are always enabled
 	for _, subtaskMeta := range subtaskMetas {
+		if syncPolicy != nil && syncPolicy.SkipCollectors && strings.Contains(strings.ToLower(subtaskMeta.Name), "collect") {
+			subtasksFlag[subtaskMeta.Name] = false
+		}
 		if subtaskMeta.Required {
 			subtasksFlag[subtaskMeta.Name] = true
 		}

backend/plugins/bamboo/models/migrationscripts/20231121_add_missing_primary_key.go

@@ -0,0 +1,59 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addMissingPrimaryKeyForBambooPlanBuildVcsRevision)(nil)
+
+type addMissingPrimaryKeyForBambooPlanBuildVcsRevision struct{}
+
+func (*addMissingPrimaryKeyForBambooPlanBuildVcsRevision) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+	if err := db.DropTables(&BambooPlanBuildVcsRevision20231121{}); err != nil {
+		return err
+	}
+	return db.AutoMigrate(&BambooPlanBuildVcsRevision20231121{})
+}
+
+func (*addMissingPrimaryKeyForBambooPlanBuildVcsRevision) Version() uint64 {
+	return 20231121101000
+}
+
+func (*addMissingPrimaryKeyForBambooPlanBuildVcsRevision) Name() string {
+	return "add missing primary key plan_result_key for table _tool_bamboo_plan_build_commits"
+}
+
+type BambooPlanBuildVcsRevision20231121 struct {
+	ConnectionId   uint64 `gorm:"primaryKey"`
+	PlanBuildKey   string `gorm:"primaryKey"`
+	PlanResultKey  string `gorm:"primaryKey"`
+	RepositoryId   int
+	RepositoryName string `json:"repositoryName"`
+	VcsRevisionKey string `gorm:"primaryKey"`
+	archived.NoPKModel
+}
+
+func (BambooPlanBuildVcsRevision20231121) TableName() string {
+	return "_tool_bamboo_plan_build_commits"
+}

backend/plugins/bamboo/models/migrationscripts/register.go

@@ -35,5 +35,6 @@ func All() []plugin.MigrationScript {
 		new(renameToolBambooDeployBuild20230919),
 		new(renameToolBambooDeployEnvironments20230919),
 		new(renameMultiBambooRawTables20230920),
+		new(addMissingPrimaryKeyForBambooPlanBuildVcsRevision),
 	}
 }

backend/plugins/bitbucket/models/migrationscripts/20231123_add_primary_key_for_bitbucket_pipeline_steps.go

@@ -0,0 +1,70 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"time"
+)
+
+var _ plugin.MigrationScript = (*reCreatBitBucketPipelineSteps)(nil)
+
+type bitbucketPipelineStep20231123 struct {
+	ConnectionId      uint64 `gorm:"primaryKey"`
+	BitbucketId       string `gorm:"primaryKey"`
+	PipelineId        string `gorm:"type:varchar(255)"`
+	Name              string `gorm:"type:varchar(255)"`
+	Trigger           string `gorm:"type:varchar(255)"`
+	State             string `gorm:"type:varchar(255)"`
+	Result            string `gorm:"type:varchar(255)"`
+	RepoId            string `gorm:"type:varchar(255)"`
+	MaxTime           int
+	StartedOn         *time.Time
+	CompletedOn       *time.Time
+	DurationInSeconds int
+	BuildSecondsUsed  int
+	RunNumber         int
+	Type              string `gorm:"type:varchar(255)"`
+	Environment       string `gorm:"type:varchar(255)"`
+	archived.NoPKModel
+}
+
+func (bitbucketPipelineStep20231123) TableName() string {
+	return "_tool_bitbucket_pipeline_steps"
+}
+
+type reCreatBitBucketPipelineSteps struct{}
+
+func (script *reCreatBitBucketPipelineSteps) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+	if err := db.DropTables(bitbucketPipelineStep20231123{}.TableName()); err != nil {
+		return err
+	}
+	return db.AutoMigrate(&bitbucketPipelineStep20231123{})
+}
+
+func (*reCreatBitBucketPipelineSteps) Version() uint64 {
+	return 20231123160001
+}
+
+func (script *reCreatBitBucketPipelineSteps) Name() string {
+	return "re create _tool_bitbucket_pipeline_steps, make sure primary keys exist."
+}

backend/plugins/bitbucket/models/migrationscripts/register.go

@@ -39,5 +39,6 @@ func All() []plugin.MigrationScript {
 		new(renameTr2ScopeConfig),
 		new(addRawParamTableForScope),
 		new(addBuildNumberToPipelines),
+		new(reCreatBitBucketPipelineSteps),
 	}
 }

backend/plugins/teambition/models/migrationscripts/20231123_recreate_teambition_connections.go

@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*reCreateTeambitionConnections)(nil)
+
+type teambitionConnection20231123 struct {
+	Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"`
+	archived.Model
+	archived.RestConnection `mapstructure:",squash"`
+	TenantId                string `mapstructure:"tenantId" validate:"required" json:"tenantId"`
+	TenantType              string `mapstructure:"tenantType" validate:"required" json:"tenantType"`
+}
+
+func (teambitionConnection20231123) TableName() string {
+	return "_tool_teambition_connections"
+}
+
+type reCreateTeambitionConnections struct{}
+
+func (*reCreateTeambitionConnections) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+	if err := db.DropTables(teambitionConnection20231123{}.TableName()); err != nil {
+		return err
+	}
+	return db.AutoMigrate(&teambitionConnection20231123{})
+}
+
+func (*reCreateTeambitionConnections) Version() uint64 {
+	return 20231123160000
+}
+
+func (*reCreateTeambitionConnections) Name() string {
+	return "recreate teambition connection table"
+}

backend/plugins/teambition/models/migrationscripts/register.go

@@ -23,5 +23,6 @@ import "github.com/apache/incubator-devlake/core/plugin"
 func All() []plugin.MigrationScript {
 	return []plugin.MigrationScript{
 		new(addInitTables),
+		new(reCreateTeambitionConnections),
 	}
 }

backend/server/services/blueprint_makeplan_v200.go

@@ -73,18 +73,7 @@ func GeneratePlanJsonV200(
 	// skip collectors
 	if skipCollectors {
 		for i, plan := range sourcePlans {
-			for j, stage := range plan {
-				for k, task := range stage {
-					newSubtasks := make([]string, 0, len(task.Subtasks))
-					for _, subtask := range task.Subtasks {
-						if !strings.Contains(strings.ToLower(subtask), "collect") {
-							newSubtasks = append(newSubtasks, subtask)
-						}
-					}
-					task.Subtasks = newSubtasks
-					sourcePlans[i][j][k] = task
-				}
-			}
+			sourcePlans[i] = removeCollectorTasks(plan)
 		}
 
 		// remove gitextractor plugin if it's not the only task
@@ -150,3 +139,19 @@ func GeneratePlanJsonV200(
 	)
 	return plan, err
 }
+
+func removeCollectorTasks(plan coreModels.PipelinePlan) coreModels.PipelinePlan {
+	for j, stage := range plan {
+		for k, task := range stage {
+			newSubtasks := make([]string, 0, len(task.Subtasks))
+			for _, subtask := range task.Subtasks {
+				if !strings.Contains(strings.ToLower(subtask), "collect") {
+					newSubtasks = append(newSubtasks, subtask)
+				}
+			}
+			task.Subtasks = newSubtasks
+			plan[j][k] = task
+		}
+	}
+	return plan
+}

backend/server/services/blueprint_test.go

@@ -92,3 +92,58 @@ func TestParallelizePipelineTasks(t *testing.T) {
 		ParallelizePipelinePlans(plan1, plan2, plan3),
 	)
 }
+
+func TestRemoveCollectorTasks(t *testing.T) {
+	plan1 := coreModels.PipelinePlan{
+		{
+			{
+				Plugin: "github",
+				Subtasks: []string{
+					"CollectApiPipelines",
+					"ExtractApiPipelines",
+					"collectApiPipelineDetails",
+					"extractApiPipelineDetails",
+					"collectApiJobs",
+					"extractApiJobs",
+					"collectAccounts",
+					"extractAccounts",
+					"ConvertAccounts",
+					"convertApiProject",
+					"convertPipelines",
+					"convertPipelineCommits",
+					"convertJobs",
+				},
+			},
+		},
+		{
+			{
+				Plugin:   "starrocks",
+				Subtasks: []string{},
+			},
+		},
+	}
+	assert.Equal(t, coreModels.PipelinePlan{
+		{
+			{
+				Plugin: "github",
+				Subtasks: []string{
+					"ExtractApiPipelines",
+					"extractApiPipelineDetails",
+					"extractApiJobs",
+					"extractAccounts",
+					"ConvertAccounts",
+					"convertApiProject",
+					"convertPipelines",
+					"convertPipelineCommits",
+					"convertJobs",
+				},
+			},
+		},
+		{
+			{
+				Plugin:   "starrocks",
+				Subtasks: []string{},
+			},
+		},
+	}, removeCollectorTasks(plan1))
+}

backend/server/services/locking.go

@@ -44,11 +44,12 @@ func lockDatabase() {
 	}
 	errors.Must(db.Create(lockingHistory))
 	// 2. obtain the lock: using a never released transaction
-	// This prevent multiple devlake instances from sharing the same database by locking the migration history table
+	// This prevents multiple devlake instances from sharing the same database by locking the migration history table
 	// However, it would not work if any older devlake instances were already using the database.
 	lockingTx = db.Begin()
 	c := make(chan bool, 1)
 	go func() {
+		errors.Must(lockingTx.DropTables(models.LockingStub{}.TableName()))
 		errors.Must(lockingTx.AutoMigrate(&models.LockingStub{}))
 		errors.Must(lockingTx.LockTables(dal.LockTables{{Table: "_devlake_locking_stub", Exclusive: true}}))
 		lockingHistory.Succeeded = true

config-ui/package.json

@@ -23,6 +23,7 @@
   },
   "dependencies": {
     "@ahooksjs/use-url-state": "^3.5.1",
+    "@ant-design/icons": "^5.2.6",
     "@blueprintjs/core": "^5.3.0",
     "@blueprintjs/datetime2": "^1.0.10",
     "@blueprintjs/popover2": "^2.0.10",
@@ -66,6 +67,7 @@
     "lint-staged": "^13.1.0",
     "prettier": "^2.7.1",
     "typescript": "^4.9.4",
-    "vite": "^4.2.3"
+    "vite": "^4.2.3",
+    "vite-plugin-svgr": "^4.2.0"
   }
 }