From e87eb93cb782c80b203842967f4b093dddd3413c Mon Sep 17 00:00:00 2001 From: Suyash Choudhary <57896905+sssash18@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:31:41 +0530 Subject: [PATCH] WIP : local cloud provider --- NOTES.md | 16 ++ README.md | 1 + .../virtual/virtual_cloud_provider.go | 218 ++++++++++++++++++ .../cloudprovider/virtual/virtual_manager.go | 1 + cluster-autoscaler/go.mod | 4 + cluster-autoscaler/go.sum | 8 + .../hack/get-gardener-cluster-info.sh | 0 7 files changed, 248 insertions(+) create mode 100644 NOTES.md create mode 100644 cluster-autoscaler/cloudprovider/virtual/virtual_cloud_provider.go create mode 100644 cluster-autoscaler/cloudprovider/virtual/virtual_manager.go create mode 100644 cluster-autoscaler/hack/get-gardener-cluster-info.sh diff --git a/NOTES.md b/NOTES.md new file mode 100644 index 000000000000..a8b1a15f5946 --- /dev/null +++ b/NOTES.md @@ -0,0 +1,16 @@ +### Creating Nodes in virtual cluster + +#### Option 1 +- this CA binary will be passed an additional flag `gardener-shoot-name`. +- It can connect to the shoot and get the shoot yaml +- It knows about nodegroups, workerpools, etc. +- This info can be used for creating node objects during scaleups in virtual cluster. + +#### Option 2 +- We have a small script - `grab_cluster_info`. +- This will connect to gardener cluster and download relevant yaml files (shoot yaml,ca yaml,mcd yaml,nodes yaml). +- CA can be configured with path `gardener-cluster-info=`. +- local provider can now launch virtual nodes using this info. + +#### Option 3 +- CA can be configured with a custom param `gardener-cluster-hist=` . \ No newline at end of file diff --git a/README.md b/README.md index b711d4bb25aa..3de4564862e4 100644 --- a/README.md +++ b/README.md @@ -45,3 +45,4 @@ Please refer to Kubernetes [Github workflow guide] for more details. [GoDoc]: https://godoc.org/k8s.io/autoscaler [GoDoc Widget]: https://godoc.org/k8s.io/autoscaler?status.svg [Github workflow guide]: https://github.com/kubernetes/community/blob/master/contributors/guide/github-workflow.md + diff --git a/cluster-autoscaler/cloudprovider/virtual/virtual_cloud_provider.go b/cluster-autoscaler/cloudprovider/virtual/virtual_cloud_provider.go new file mode 100644 index 000000000000..c3e529599ab1 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/virtual/virtual_cloud_provider.go @@ -0,0 +1,218 @@ +package virtual + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/config" + "k8s.io/autoscaler/cluster-autoscaler/utils/errors" + "k8s.io/klog/v2" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" + "os" +) + +type VirtualNodeGroup struct { + Name string + poolName string + zone string + targetSize int + minSize int + maxSize int +} + +var _ cloudprovider.NodeGroup = (*VirtualNodeGroup)(nil) + +type VirtualCloudProvider struct { + clusterInfo *clusterInfo +} + +type VirtualWorkerPool struct { + MachineType string + Architecture string + Minimum int + Maximum int + MaxSurge intstr.IntOrString + MaxUnavailable intstr.IntOrString + Zones []string +} + +type VirtualMachineDeployment struct { +} + +type clusterInfo struct { + nodeTemplates map[string]corev1.Node + nodeGroups map[string]VirtualNodeGroup + workerPools map[string]VirtualWorkerPool +} + +func BuildVirtual(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider { + + if opts.CloudProviderName != "virtual" { + return nil + } + + clusterInfoPath := os.Getenv("GARDENER_CLUSTER_INFO") + clusterHistoryPath := os.Getenv("GARDENER_CLUSTER_HISTORY") + if clusterHistoryPath == "" && clusterInfoPath == "" { + klog.Fatalf("cannot create virtual provider one of env var GARDENER_CLUSTER_INFO or GARDENER_CLUSTER_HISTORY needs to be set.") + return nil + } + + if clusterInfoPath != "" { + readInitClusterInfo(clusterInfoPath) + } + return nil + +} + +func readInitClusterInfo(clusterInfoPath string) clusterInfo { + +} + +func (v VirtualCloudProvider) Name() string { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) NodeGroups() []cloudprovider.NodeGroup { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) NodeGroupForNode(node *corev1.Node) (cloudprovider.NodeGroup, error) { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) HasInstance(node *corev1.Node) (bool, error) { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) Pricing() (cloudprovider.PricingModel, errors.AutoscalerError) { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) GetAvailableMachineTypes() ([]string, error) { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) NewNodeGroup(machineType string, labels map[string]string, systemLabels map[string]string, taints []corev1.Taint, extraResources map[string]resource.Quantity) (cloudprovider.NodeGroup, error) { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) GetResourceLimiter() (*cloudprovider.ResourceLimiter, error) { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) GPULabel() string { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) GetAvailableGPUTypes() map[string]struct{} { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) GetNodeGpuConfig(node *corev1.Node) *cloudprovider.GpuConfig { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) Cleanup() error { + //TODO implement me + panic("implement me") +} + +func (v VirtualCloudProvider) Refresh() error { + //TODO implement me + panic("implement me") +} + +var _ cloudprovider.CloudProvider = (*VirtualCloudProvider)(nil) + +func (v VirtualNodeGroup) MaxSize() int { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) MinSize() int { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) TargetSize() (int, error) { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) IncreaseSize(delta int) error { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) AtomicIncreaseSize(delta int) error { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) DeleteNodes(nodes []*corev1.Node) error { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) DecreaseTargetSize(delta int) error { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) Id() string { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) Debug() string { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) Nodes() ([]cloudprovider.Instance, error) { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) TemplateNodeInfo() (*schedulerframework.NodeInfo, error) { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) Exist() bool { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) Create() (cloudprovider.NodeGroup, error) { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) Delete() error { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) Autoprovisioned() bool { + //TODO implement me + panic("implement me") +} + +func (v VirtualNodeGroup) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*config.NodeGroupAutoscalingOptions, error) { + //TODO implement me + panic("implement me") +} diff --git a/cluster-autoscaler/cloudprovider/virtual/virtual_manager.go b/cluster-autoscaler/cloudprovider/virtual/virtual_manager.go new file mode 100644 index 000000000000..abf03f1d197b --- /dev/null +++ b/cluster-autoscaler/cloudprovider/virtual/virtual_manager.go @@ -0,0 +1 @@ +package virtual diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index 6df3d30d68a4..d7141017e11a 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -25,12 +25,14 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/go-querystring v1.0.0 github.com/google/uuid v1.6.0 + github.com/jmattheis/goverter v1.4.0 github.com/jmespath/go-jmespath v0.4.0 github.com/json-iterator/go v1.1.12 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 + github.com/vburenin/ifacemaker v1.2.1 go.uber.org/mock v0.4.0 golang.org/x/crypto v0.19.0 golang.org/x/net v0.21.0 @@ -90,6 +92,7 @@ require ( github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/dave/jennifer v1.6.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/reference v0.5.0 // indirect @@ -125,6 +128,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jessevdk/go-flags v1.4.1-0.20181029123624-5de817a9aa20 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/karrick/godirwalk v1.17.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/cluster-autoscaler/go.sum b/cluster-autoscaler/go.sum index 24f4842b9b92..6498ee47cddf 100644 --- a/cluster-autoscaler/go.sum +++ b/cluster-autoscaler/go.sum @@ -196,6 +196,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/dave/jennifer v1.6.0 h1:MQ/6emI2xM7wt0tJzJzyUik2Q3Tcn2eE0vtYgh4GPVI= +github.com/dave/jennifer v1.6.0/go.mod h1:AxTG893FiZKqxy3FP1kL80VMshSMuz2G+EgvszgGRnk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -414,6 +416,10 @@ github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v1.4.1-0.20181029123624-5de817a9aa20 h1:dAOsPLhnBzIyxu0VvmnKjlNcIlgMK+erD6VRHDtweMI= +github.com/jessevdk/go-flags v1.4.1-0.20181029123624-5de817a9aa20/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmattheis/goverter v1.4.0 h1:SrboBYMpGkj1XSgFhWwqzdP024zIa1+58YzUm+0jcBE= +github.com/jmattheis/goverter v1.4.0/go.mod h1:iVIl/4qItWjWj2g3vjouGoYensJbRqDHpzlEVMHHFeY= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -579,6 +585,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vburenin/ifacemaker v1.2.1 h1:3Vq8B/bfBgjWTkv+jDg4dVL1KHt3k1K4lO7XRxYA2sk= +github.com/vburenin/ifacemaker v1.2.1/go.mod h1:5WqrzX2aD7/hi+okBjcaEQJMg4lDGrpuEX3B8L4Wgrs= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= diff --git a/cluster-autoscaler/hack/get-gardener-cluster-info.sh b/cluster-autoscaler/hack/get-gardener-cluster-info.sh new file mode 100644 index 000000000000..e69de29bb2d1