refactor: update cost image and use builtin chart

pack/server/image/Dockerfile

@@ -152,6 +152,12 @@ COPY --from=sealio/casdoor:v1.344.0-seal.1 \
     /casdoor \
     /usr/bin/
 
+# finops tools
+ENV IMAGE_OPENCOST="sealio/mirrored-kubecost-cost-model:v1.105.2" \
+    CHARTS_DIR="/var/lib/walrus/manifests/charts"
+RUN mkdir -p ${CHARTS_DIR}
+ADD https://github.com/prometheus-community/helm-charts/releases/download/prometheus-24.0.0/prometheus-24.0.0.tgz ${CHARTS_DIR}/prometheus.tgz
+
 # walrus cli
 RUN set -eux; \
     mkdir -p /var/lib/walrus/cli

pkg/apis/connector/basic.go

@@ -251,7 +251,7 @@ func applyFinOps(mc model.ClientSet, conn *model.Connector, reinstall bool) erro
 		defer cancel()
 
 		// Deploy tools.
-		err := deployer.DeployCostTools(ctx, conn, reinstall)
+		err := deployer.DeployCostTools(ctx, mc, conn, reinstall)
 		if err != nil {
 			// Log instead of return error, then continue to sync the final status to connector.
 			logger.Errorf("error ensuring cost tools for connector %q: %v", conn.ID, err)

pkg/costs/deployer/cost_tools.go

@@ -5,9 +5,9 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"os"
 	"path"
 
-	"helm.sh/helm/v3/pkg/repo"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	appv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
@@ -16,6 +16,7 @@ import (
 	"github.com/seal-io/walrus/pkg/dao/model"
 	"github.com/seal-io/walrus/pkg/dao/types"
 	opk8s "github.com/seal-io/walrus/pkg/operator/k8s"
+	"github.com/seal-io/walrus/pkg/settings"
 	"github.com/seal-io/walrus/utils/log"
 )
 
@@ -28,9 +29,16 @@ const (
 	pathOpencostRefreshPricing = "/refreshPricing"
 )
 
+var (
+	prometheusChartTgz = "prometheus.tgz"
+	imageOpencost      = path.Join(os.Getenv("IMAGE_OPENCOST"))
+)
+
 const (
-	defaultPrometheusChartTgz = "prometheus-19.6.1.tgz"
-	defaultPrometheusRepo     = "https://prometheus-community.github.io/helm-charts"
+	repositoryServer       = "sealio/mirrored-prometheus"
+	repositoryNodeExporter = "sealio/mirrored-node-exporter"
+	repositoryKubeState    = "sealio/mirrored-kube-state-metrics"
+	repositoryReload       = "sealio/mirrored-prometheus-config-reloader"
 )
 
 var pathServiceProxy = fmt.Sprintf("/api/v1/namespaces/%s/services/http:%s:9003/proxy",
@@ -41,9 +49,10 @@ type input struct {
 	Namespace          string
 	ClusterID          string
 	PrometheusEndpoint string
+	Image              string
 }
 
-func DeployCostTools(ctx context.Context, conn *model.Connector, replace bool) error {
+func DeployCostTools(ctx context.Context, mc model.ClientSet, conn *model.Connector, replace bool) error {
 	log.WithName("cost").Debugf("deploying cost tools for connector %s", conn.Name)
 
 	apiConfig, kubeconfig, err := opk8s.LoadApiConfig(*conn)
@@ -56,9 +65,12 @@ func DeployCostTools(ctx context.Context, conn *model.Connector, replace bool) e
 		return fmt.Errorf("error create deployer: %w", err)
 	}
 
-	clusterName := apiConfig.CurrentContext
+	var (
+		clusterName   = apiConfig.CurrentContext
+		imageRegistry = settings.ImageRegistry.ShouldValue(ctx, mc)
+	)
 
-	yaml, err := opencost(clusterName)
+	yaml, err := opencost(clusterName, imageRegistry)
 	if err != nil {
 		return err
 	}
@@ -67,7 +79,7 @@ func DeployCostTools(ctx context.Context, conn *model.Connector, replace bool) e
 		return err
 	}
 
-	app, err := prometheus()
+	app, err := prometheus(imageRegistry)
 	if err != nil {
 		return err
 	}
@@ -126,12 +138,15 @@ func CostToolsStatus(ctx context.Context, conn *model.Connector) error {
 	return nil
 }
 
-func opencost(clusterName string) ([]byte, error) {
+func opencost(clusterName, imageRegistry string) ([]byte, error) {
+	image := path.Join(imageRegistry, imageOpencost)
+
 	data := input{
 		Name:               NameOpencost,
 		Namespace:          types.WalrusSystemNamespace,
 		ClusterID:          clusterName,
 		PrometheusEndpoint: fmt.Sprintf("http://%s-server.%s.svc:80", NamePrometheus, types.WalrusSystemNamespace),
+		Image:              image,
 	}
 
 	buf := &bytes.Buffer{}
@@ -185,37 +200,62 @@ func opencostRefreshPricingURL(restCfg *rest.Config) (string, error) {
 	return u.String(), nil
 }
 
-func prometheus() (*ChartApp, error) {
+func prometheus(imageRegistry string) (*ChartApp, error) {
 	scrape, err := opencostScrape()
 	if err != nil {
 		return nil, err
 	}
 
+	imageConfig := func(repo string) map[string]any {
+		cfg := map[string]any{
+			"registry": imageRegistry,
+		}
+
+		if repo != "" {
+			cfg["repository"] = repo
+		}
+
+		return cfg
+	}
+
 	values := map[string]any{
 		"prometheus-pushgateway": map[string]any{
 			"enabled": false,
 		},
 		"alertmanager": map[string]any{
 			"enabled": false,
 		},
+
+		"kube-state-metrics": map[string]any{
+			"image": imageConfig(repositoryKubeState),
+		},
+		"prometheus-node-exporter": map[string]any{
+			"image": imageConfig(repositoryNodeExporter),
+		},
+		"extraScrapeConfigs": scrape,
+
+		// Configmap reload and prometheus only support include registry in repository.
+		"configmapReload": map[string]any{
+			"prometheus": map[string]any{
+				"image": map[string]any{
+					"repository": path.Join(imageRegistry, repositoryReload),
+				},
+			},
+		},
 		"server": map[string]any{
 			"persistentVolume": map[string]any{
 				"enabled": false,
 			},
+			"image": map[string]any{
+				"repository": path.Join(imageRegistry, repositoryServer),
+			},
 		},
-		"extraScrapeConfigs": scrape,
-	}
-
-	entry := &repo.Entry{
-		Name: "prometheus",
-		URL:  defaultPrometheusRepo,
 	}
 
 	return &ChartApp{
 		Name:         NamePrometheus,
 		Namespace:    types.WalrusSystemNamespace,
-		ChartTgzName: defaultPrometheusChartTgz,
+		ChartTgzName: prometheusChartTgz,
 		Values:       values,
-		Entry:        entry,
 	}, nil
 }

pkg/costs/deployer/deploy_test.go

@@ -24,10 +24,10 @@ func TestHelm(t *testing.T) {
 	deployer, err := New(string(kubeConfigContentByte))
 	assert.Nil(t, err, "error create helm")
 
-	yaml, err := opencost("test")
+	yaml, err := opencost("test", "docker.io")
 	assert.Nil(t, err, "error create opencost yaml")
 
-	app, err := prometheus()
+	app, err := prometheus("docker.io")
 	assert.Nil(t, err, "error create prometheus app")
 
 	err = deployer.EnsureChart(app, true)

pkg/costs/deployer/deployer.go

@@ -162,12 +162,9 @@ func (d *Deployer) EnsureChart(app *ChartApp, replace bool) error {
 		}
 	}
 
-	chartTgzPath := path.Join(helm.repoCache, app.ChartTgzName)
+	chartTgzPath := path.Join(helm.chartsDir, app.ChartTgzName)
 	if _, err = os.Stat(chartTgzPath); err != nil {
-		chartTgzPath, err = helm.Download(app.Entry.URL, app.Entry.Name)
-		if err != nil {
-			return err
-		}
+		return err
 	}
 
 	if err = helm.Install(app.Name, chartTgzPath, app.Values); err != nil {

pkg/costs/deployer/helm.go

@@ -10,7 +10,6 @@ import (
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/cli"
 	"helm.sh/helm/v3/pkg/release"
-	"helm.sh/helm/v3/pkg/repo"
 
 	"github.com/seal-io/walrus/utils/log"
 )
@@ -19,20 +18,21 @@ const (
 	timeout = 5 * time.Minute
 )
 
+var chartsDir = os.Getenv("CHARTS_DIR")
+
 type ChartApp struct {
 	Name         string
 	Namespace    string
 	ChartTgzName string
 	Values       map[string]any
-	Entry        *repo.Entry
 }
 
 type Helm struct {
 	settings     *cli.EnvSettings
 	actionConfig *action.Configuration
 	kubeCfgFile  *os.File
 	namespace    string
-	repoCache    string
+	chartsDir    string
 	logger       log.Logger
 }
 
@@ -47,16 +47,6 @@ func NewHelm(namespace, kubeconfig string) (*Helm, error) {
 		return nil, err
 	}
 
-	repoPath := path.Join(storeBaseDir, "repository")
-	if err := os.MkdirAll(repoPath, 0o777); err != nil {
-		return nil, err
-	}
-
-	repoCachePath := path.Join(storeBaseDir, "repository-cache")
-	if err := os.MkdirAll(repoCachePath, 0o777); err != nil {
-		return nil, err
-	}
-
 	kubeconfigFile, err := os.CreateTemp(storeBaseDir, "kubeconfig")
 	if err != nil {
 		return nil, err
@@ -67,8 +57,6 @@ func NewHelm(namespace, kubeconfig string) (*Helm, error) {
 	}
 
 	settings := cli.New()
-	settings.RepositoryConfig = path.Join(repoPath, "repositories.yaml")
-	settings.RepositoryCache = repoCachePath
 	settings.KubeConfig = kubeconfigFile.Name()
 
 	config := action.Configuration{}
@@ -85,29 +73,11 @@ func NewHelm(namespace, kubeconfig string) (*Helm, error) {
 		actionConfig: &config,
 		kubeCfgFile:  kubeconfigFile,
 		namespace:    namespace,
-		repoCache:    repoCachePath,
+		chartsDir:    chartsDir,
 		logger:       logger,
 	}, nil
 }
 
-func (h *Helm) ChartCacheDir() string {
-	return h.repoCache
-}
-
-func (h *Helm) Download(repoURL, chartName string) (string, error) {
-	h.logger.Debugf("downloading %s from %s", chartName, repoURL)
-	chartOps := action.ChartPathOptions{
-		RepoURL: repoURL,
-	}
-
-	outputPath, err := chartOps.LocateChart(chartName, h.settings)
-	if err != nil {
-		return "", fmt.Errorf("error download chart %s:%s, %w", repoURL, chartName, err)
-	}
-
-	return outputPath, nil
-}
-
 func (h *Helm) Install(name, chartPath string, values map[string]any) error {
 	ch, err := loader.Load(chartPath)
 	if err != nil {

pkg/costs/deployer/template.go

@@ -10,18 +10,26 @@ var (
 		template.New("prometheusScrapeJob").Parse(tmplPrometheusScrapeJobContent))
 )
 
-// source: https://github.com/opencost/opencost/blob/v1.100.2/kubernetes/opencost.yaml
-var tmplOpencostContent = `apiVersion: v1
+// source: https://github.com/opencost/opencost/blob/v1.105.2/kubernetes/opencost.yaml.
+var tmplOpencostContent = `---
+
+# The namespace OpenCost will run in.
+apiVersion: v1
 kind: Namespace
 metadata:
     name: {{.Namespace}}
 ---
+
+# Service account for permissions.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: {{.Name}}
   namespace: {{.Namespace}}
 ---
+
+# Cluster role giving OpenCost to get, list, watch required resources
+# No write permissions are required.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -99,7 +107,10 @@ rules:
       - get
       - list
       - watch
+
 ---
+
+# Bind the role to the service account.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -113,6 +124,8 @@ subjects:
     name: {{.Name}}
     namespace: {{.Namespace}}
 ---
+
+# Create a deployment for a single cost model pod.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -138,7 +151,7 @@ spec:
       restartPolicy: Always
       serviceAccountName: {{.Name}}
       containers:
-        - image: quay.io/kubecost1/kubecost-cost-model:latest
+        - image: {{.Image}}
           name: opencost
           resources:
             requests:
@@ -159,7 +172,15 @@ spec:
               value: {{.Name}}
             - name: KUBECOST_NAMESPACE
               value: {{.Namespace}}
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            privileged: false
+            readOnlyRootFilesystem: true
+            runAsUser: 1001
 ---
 kind: Service
 apiVersion: v1
@@ -173,7 +194,8 @@ spec:
   ports:
     - name: {{.Name}}
       port: 9003
-      targetPort: 9003`
+      targetPort: 9003
+---`
 
 // source: https://github.com/opencost/opencost/blob/v1.100.2/kubernetes/prometheus/extraScrapeConfigs.yaml
 var tmplPrometheusScrapeJobContent = `- job_name: {{.Name}}

pkg/settings/settings.go

@@ -123,6 +123,12 @@ var (
 		editable,
 		initializeFrom("true"),
 		modifyWith(notBlank))
+	// ImageRegistry config the image registry for seal tools, like finOps tools.
+	ImageRegistry = newValue(
+		"ImageRegistry",
+		editable,
+		initializeFrom("docker.io"),
+		modifyWith(notBlank))
 )
 
 // the built-in settings for server cron jobs.