Skip to content

Commit

Permalink
Merge pull request #6630 from ionos-cloud/update-ionos-sdk-1.28
Browse files Browse the repository at this point in the history
Backport #6617 ionoscloud: Update ionos-cloud sdk-go and add metrics into CA 1.28
k8s-ci-robot authored Mar 14, 2024
2 parents 1cdd2a2 + 34108be commit 67895bf
Showing 230 changed files with 25,030 additions and 12,248 deletions.
13 changes: 12 additions & 1 deletion cluster-autoscaler/cloudprovider/ionoscloud/README.md
Original file line number Diff line number Diff line change
@@ -25,12 +25,18 @@ Store the token in a secret:
kubectl create secret generic cloud-config --from-literal=token=MY_TOKEN
```

Edit [`example-values.yaml`](./examples-values.yaml) and deploy using helm:
Edit [`example-values.yaml`](./example-values.yaml) and deploy using helm:

```console
helm install ionoscloud-cluster-autoscaler autoscaler/cluster-autoscaler -f example-values.yaml
```

### Configuration

The `IONOS_ADDITIONAL_HEADERS` environment variable can be used to configure the client to send additional headers on
each Ionos Cloud API request, such as `X-Contract-Number`. This can be useful for users with multiple contracts.
The format is a semicolon-separated list of key:value pairs, e.g. `IONOS_ADDITIONAL_HEADERS="X-Contract-Number:1234657890"`.

## Development

The unit tests use mocks generated by [mockery](https://github.com/vektra/mockery/v2). To update them run:
@@ -40,6 +46,11 @@ mockery --inpackage --testonly --case snake --boilerplate-file ../../../hack/boi
mockery --inpackage --testonly --case snake --boilerplate-file ../../../hack/boilerplate/boilerplate.generatego.txt --name IonosCloudManager
```

### Debugging

The global logging verbosity controlled by the `--v` flag is passed on to the Ionos Cloud SDK client logger.
Verbosity 5 maps to `Debug` and 7 to `Trace`. **Do not enable this in production, as it will print full request and response data.**

### Build an image

Build and push a docker image in the `cluster-autoscaler` directory:
92 changes: 34 additions & 58 deletions cluster-autoscaler/cloudprovider/ionoscloud/cache.go
Original file line number Diff line number Diff line change
@@ -18,28 +18,18 @@ package ionoscloud

import (
"sync"
"time"

"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
"k8s.io/klog/v2"
)

const nodeGroupCacheEntryTimeout = 2 * time.Minute

type nodeGroupCacheEntry struct {
data cloudprovider.NodeGroup
ts time.Time
}

var timeNow = time.Now

// IonosCache caches resources to reduce API calls.
// Cached state includes autoscaling limits, sizes and target sizes, a mapping of instances to node
// groups, and a simple lock mechanism to prevent invalid node group writes.
type IonosCache struct {
mutex sync.Mutex

nodeGroups map[string]nodeGroupCacheEntry
nodeGroups map[string]cloudprovider.NodeGroup
nodesToNodeGroups map[string]string
nodeGroupSizes map[string]int
nodeGroupTargetSizes map[string]int
@@ -49,11 +39,11 @@ type IonosCache struct {
// NewIonosCache initializes a new IonosCache.
func NewIonosCache() *IonosCache {
return &IonosCache{
nodeGroups: map[string]nodeGroupCacheEntry{},
nodesToNodeGroups: map[string]string{},
nodeGroupSizes: map[string]int{},
nodeGroupTargetSizes: map[string]int{},
nodeGroupLockTable: map[string]bool{},
nodeGroups: make(map[string]cloudprovider.NodeGroup),
nodesToNodeGroups: make(map[string]string),
nodeGroupSizes: make(map[string]int),
nodeGroupTargetSizes: make(map[string]int),
nodeGroupLockTable: make(map[string]bool),
}
}

@@ -62,15 +52,7 @@ func (cache *IonosCache) AddNodeGroup(newPool cloudprovider.NodeGroup) {
cache.mutex.Lock()
defer cache.mutex.Unlock()

cache.nodeGroups[newPool.Id()] = nodeGroupCacheEntry{data: newPool}
}

func (cache *IonosCache) removeNodesForNodeGroupNoLock(id string) {
for nodeId, nodeGroupId := range cache.nodesToNodeGroups {
if nodeGroupId == id {
delete(cache.nodesToNodeGroups, nodeId)
}
}
cache.nodeGroups[newPool.Id()] = newPool
}

// RemoveInstanceFromCache deletes an instance and its respective mapping to the node group from
@@ -79,38 +61,48 @@ func (cache *IonosCache) RemoveInstanceFromCache(id string) {
cache.mutex.Lock()
defer cache.mutex.Unlock()

klog.V(5).Infof("Removed instance %s from cache", id)
nodeGroupId := cache.nodesToNodeGroups[id]
delete(cache.nodesToNodeGroups, id)
cache.updateNodeGroupTimestampNoLock(nodeGroupId)
if _, ok := cache.nodesToNodeGroups[id]; ok {
delete(cache.nodesToNodeGroups, id)
klog.V(5).Infof("Removed instance %s from cache", id)
}
}

// SetInstancesCacheForNodeGroup overwrites cached instances and mappings for a node group.
func (cache *IonosCache) SetInstancesCacheForNodeGroup(id string, instances []cloudprovider.Instance) {
cache.mutex.Lock()
defer cache.mutex.Unlock()

cache.removeNodesForNodeGroupNoLock(id)
mapsDeleteFunc(cache.nodesToNodeGroups, func(_, nodeGroupID string) bool {
return nodeGroupID == id
})
cache.setInstancesCacheForNodeGroupNoLock(id, instances)
}

// placeholder for maps.DeleteFunc in go 1.21
func mapsDeleteFunc(m map[string]string, del func(k, v string) bool) {
for k, v := range m {
if del(k, v) {
delete(m, k)
}
}
}

func (cache *IonosCache) setInstancesCacheForNodeGroupNoLock(id string, instances []cloudprovider.Instance) {
for _, instance := range instances {
nodeId := convertToNodeId(instance.Id)
cache.nodesToNodeGroups[nodeId] = id
nodeID := convertToNodeID(instance.Id)
cache.nodesToNodeGroups[nodeID] = id
}
cache.updateNodeGroupTimestampNoLock(id)
}

// GetNodeGroupIds returns an unsorted list of cached node group ids.
func (cache *IonosCache) GetNodeGroupIds() []string {
// GetNodeGroupIDs returns an unsorted list of cached node group ids.
func (cache *IonosCache) GetNodeGroupIDs() []string {
cache.mutex.Lock()
defer cache.mutex.Unlock()

return cache.getNodeGroupIds()
return cache.getNodeGroupIDs()
}

func (cache *IonosCache) getNodeGroupIds() []string {
func (cache *IonosCache) getNodeGroupIDs() []string {
ids := make([]string, 0, len(cache.nodeGroups))
for id := range cache.nodeGroups {
ids = append(ids, id)
@@ -125,26 +117,26 @@ func (cache *IonosCache) GetNodeGroups() []cloudprovider.NodeGroup {

nodeGroups := make([]cloudprovider.NodeGroup, 0, len(cache.nodeGroups))
for id := range cache.nodeGroups {
nodeGroups = append(nodeGroups, cache.nodeGroups[id].data)
nodeGroups = append(nodeGroups, cache.nodeGroups[id])
}
return nodeGroups
}

// GetNodeGroupForNode returns the node group for the given node.
// Returns nil if either the mapping or the node group is not cached.
func (cache *IonosCache) GetNodeGroupForNode(nodeId string) cloudprovider.NodeGroup {
func (cache *IonosCache) GetNodeGroupForNode(nodeID string) cloudprovider.NodeGroup {
cache.mutex.Lock()
defer cache.mutex.Unlock()

id, found := cache.nodesToNodeGroups[nodeId]
nodeGroupID, found := cache.nodesToNodeGroups[nodeID]
if !found {
return nil
}
entry, found := cache.nodeGroups[id]
nodeGroup, found := cache.nodeGroups[nodeGroupID]
if !found {
return nil
}
return entry.data
return nodeGroup
}

// TryLockNodeGroup tries to write a node group lock entry.
@@ -219,19 +211,3 @@ func (cache *IonosCache) InvalidateNodeGroupTargetSize(id string) {

delete(cache.nodeGroupTargetSizes, id)
}

// NodeGroupNeedsRefresh returns true when the instances for the given node group have not been
// updated for more than 2 minutes.
func (cache *IonosCache) NodeGroupNeedsRefresh(id string) bool {
cache.mutex.Lock()
defer cache.mutex.Unlock()

return timeNow().Sub(cache.nodeGroups[id].ts) > nodeGroupCacheEntryTimeout
}

func (cache *IonosCache) updateNodeGroupTimestampNoLock(id string) {
if entry, ok := cache.nodeGroups[id]; ok {
entry.ts = timeNow()
cache.nodeGroups[id] = entry
}
}
35 changes: 4 additions & 31 deletions cluster-autoscaler/cloudprovider/ionoscloud/cache_test.go
Original file line number Diff line number Diff line change
@@ -18,16 +18,11 @@ package ionoscloud

import (
"testing"
"time"

"github.com/stretchr/testify/require"
"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
)

func newCacheEntry(data cloudprovider.NodeGroup, ts time.Time) nodeGroupCacheEntry {
return nodeGroupCacheEntry{data: data, ts: ts}
}

func TestCache_AddNodeGroup(t *testing.T) {
cache := NewIonosCache()
require.Empty(t, cache.GetNodeGroups())
@@ -36,17 +31,14 @@ func TestCache_AddNodeGroup(t *testing.T) {
}

func TestCache_RemoveInstanceFromCache(t *testing.T) {
firstTime := timeNow().Add(-2*time.Minute - 1*time.Second)
cache := NewIonosCache()
cache.nodeGroups["2"] = newCacheEntry(&nodePool{id: "2"}, firstTime)
cache.nodeGroups["2"] = &nodePool{id: "2"}
cache.nodesToNodeGroups["b2"] = "2"

require.NotNil(t, cache.GetNodeGroupForNode("b2"))
require.True(t, cache.NodeGroupNeedsRefresh("2"))

cache.RemoveInstanceFromCache("b2")
require.Nil(t, cache.GetNodeGroupForNode("b2"))
require.False(t, cache.NodeGroupNeedsRefresh("2"))
}

func TestCache_SetInstancesCacheForNodeGroup(t *testing.T) {
@@ -65,11 +57,11 @@ func TestCache_SetInstancesCacheForNodeGroup(t *testing.T) {

func TestCache_GetNodeGroupIDs(t *testing.T) {
cache := NewIonosCache()
require.Empty(t, cache.GetNodeGroupIds())
require.Empty(t, cache.GetNodeGroupIDs())
cache.AddNodeGroup(&nodePool{id: "1"})
require.Equal(t, []string{"1"}, cache.GetNodeGroupIds())
require.Equal(t, []string{"1"}, cache.GetNodeGroupIDs())
cache.AddNodeGroup(&nodePool{id: "2"})
require.ElementsMatch(t, []string{"1", "2"}, cache.GetNodeGroupIds())
require.ElementsMatch(t, []string{"1", "2"}, cache.GetNodeGroupIDs())
}

func TestCache_GetNodeGroups(t *testing.T) {
@@ -139,22 +131,3 @@ func TestCache_GetSetNodeGroupTargetSize(t *testing.T) {
require.False(t, found)
require.Zero(t, size)
}

func TestCache_NodeGroupNeedsRefresh(t *testing.T) {
fixedTime := time.Now().Round(time.Second)
timeNow = func() time.Time { return fixedTime }
defer func() { timeNow = time.Now }()

cache := NewIonosCache()
require.True(t, cache.NodeGroupNeedsRefresh("test"))

cache.AddNodeGroup(&nodePool{id: "test"})
require.True(t, cache.NodeGroupNeedsRefresh("test"))
cache.SetInstancesCacheForNodeGroup("test", nil)
require.False(t, cache.NodeGroupNeedsRefresh("test"))

timeNow = func() time.Time { return fixedTime.Add(nodeGroupCacheEntryTimeout) }
require.False(t, cache.NodeGroupNeedsRefresh("test"))
timeNow = func() time.Time { return fixedTime.Add(nodeGroupCacheEntryTimeout + 1*time.Second) }
require.True(t, cache.NodeGroupNeedsRefresh("test"))
}
158 changes: 71 additions & 87 deletions cluster-autoscaler/cloudprovider/ionoscloud/client.go
Original file line number Diff line number Diff line change
@@ -20,23 +20,15 @@ import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"time"

"k8s.io/apimachinery/pkg/util/wait"
ionos "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/ionoscloud/ionos-cloud-sdk-go"
"k8s.io/klog/v2"
"k8s.io/utils/pointer"
)

const (
// K8sStateActive indicates that the cluster/nodepool resource is active.
K8sStateActive = "ACTIVE"
// K8sStateUpdating indicates that the cluster/nodepool resource is being updated.
K8sStateUpdating = "UPDATING"
)

const (
@@ -65,87 +57,72 @@ type APIClient interface {
}

// NewAPIClient creates a new IonosCloud API client.
func NewAPIClient(token, endpoint, userAgent string, insecure bool) APIClient {
config := ionos.NewConfiguration("", "", token, endpoint)
func NewAPIClient(cfg *Config, userAgent string) APIClient {
config := ionos.NewConfiguration("", "", cfg.Token, cfg.Endpoint)
if userAgent != "" {
config.UserAgent = userAgent
}
if insecure {
if cfg.Insecure {
config.HTTPClient = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint: gosec
},
}
}
config.Debug = klog.V(6).Enabled()
for key, value := range cfg.AdditionalHeaders {
config.AddDefaultHeader(key, value)
}

setLogLevel(config)
config.Logger = klogAdapter{}
// Depth > 0 is only important for listing resources. All other autoscaling related requests don't need it
config.SetDepth(0)
client := ionos.NewAPIClient(config)
return client.KubernetesApi
}

// AutoscalingClient is a client abstraction used for autoscaling.
type AutoscalingClient struct {
clientProvider
clusterId string
pollTimeout time.Duration
pollInterval time.Duration
}

// NewAutoscalingClient contructs a new autoscaling client.
func NewAutoscalingClient(config *Config, userAgent string) (*AutoscalingClient, error) {
c := &AutoscalingClient{
clientProvider: newClientProvider(config, userAgent),
clusterId: config.ClusterId,
pollTimeout: config.PollTimeout,
pollInterval: config.PollInterval,
}
return c, nil
}

func newClientProvider(config *Config, userAgent string) clientProvider {
if config.Token != "" {
return defaultClientProvider{
token: config.Token,
userAgent: userAgent,
}
}
return customClientProvider{
cloudConfigDir: config.TokensPath,
endpoint: config.Endpoint,
userAgent: userAgent,
insecure: config.Insecure,
func setLogLevel(config *ionos.Configuration) {
switch {
case klog.V(7).Enabled():
config.LogLevel = ionos.Trace
case klog.V(5).Enabled():
config.LogLevel = ionos.Debug
}
}

// clientProvider initializes an authenticated Ionos Cloud API client using pre-configured values
type clientProvider interface {
GetClient() (APIClient, error)
type klogAdapter struct{}

func (klogAdapter) Printf(format string, args ...interface{}) {
klog.InfofDepth(1, "IONOSLOG "+format, args...)
}

type defaultClientProvider struct {
token string
// AutoscalingClient is a client abstraction used for autoscaling.
type AutoscalingClient struct {
client APIClient
cfg *Config
userAgent string
}

func (p defaultClientProvider) GetClient() (APIClient, error) {
return NewAPIClient(p.token, "", p.userAgent, false), nil
// NewAutoscalingClient contructs a new autoscaling client.
func NewAutoscalingClient(config *Config, userAgent string) *AutoscalingClient {
c := &AutoscalingClient{cfg: config, userAgent: userAgent}
if config.Token != "" {
c.client = NewAPIClient(config, userAgent)
}
return c
}

type customClientProvider struct {
cloudConfigDir string
endpoint string
userAgent string
insecure bool
}
func (c *AutoscalingClient) getClient() (APIClient, error) {
if c.client != nil {
return c.client, nil
}

func (p customClientProvider) GetClient() (APIClient, error) {
files, err := filepath.Glob(filepath.Join(p.cloudConfigDir, "[a-zA-Z0-9]*"))
files, err := filepath.Glob(filepath.Join(c.cfg.TokensPath, "[a-zA-Z0-9]*"))
if err != nil {
return nil, err
}
if len(files) == 0 {
return nil, fmt.Errorf("missing cloud config")
return nil, errors.New("missing cloud config")
}
data, err := os.ReadFile(files[0])
if err != nil {
@@ -160,80 +137,87 @@ func (p customClientProvider) GetClient() (APIClient, error) {
if len(cloudConfig.Tokens) == 0 {
return nil, fmt.Errorf("missing tokens for cloud config %s", filepath.Base(files[0]))
}
return NewAPIClient(cloudConfig.Tokens[0], p.endpoint, p.userAgent, p.insecure), nil
cfg := *c.cfg
cfg.Token = cloudConfig.Tokens[0]
return NewAPIClient(&cfg, c.userAgent), nil
}

// GetNodePool gets a node pool.
func (c *AutoscalingClient) GetNodePool(id string) (*ionos.KubernetesNodePool, error) {
client, err := c.GetClient()
client, err := c.getClient()
if err != nil {
return nil, err
}
req := client.K8sNodepoolsFindById(context.Background(), c.clusterId, id)
nodepool, _, err := client.K8sNodepoolsFindByIdExecute(req)
req := client.K8sNodepoolsFindById(context.Background(), c.cfg.ClusterID, id)
nodepool, resp, err := client.K8sNodepoolsFindByIdExecute(req)
registerRequest("GetNodePool", resp, err)
if err != nil {
return nil, err
}
return &nodepool, nil
}

func resizeRequestBody(targetSize int) ionos.KubernetesNodePoolForPut {
func resizeRequestBody(targetSize int32) ionos.KubernetesNodePoolForPut {
return ionos.KubernetesNodePoolForPut{
Properties: &ionos.KubernetesNodePoolPropertiesForPut{
NodeCount: pointer.Int32Ptr(int32(targetSize)),
NodeCount: &targetSize,
},
}
}

// ResizeNodePool sets the target size of a node pool and starts the resize process.
// The node pool target size cannot be changed until this operation finishes.
func (c *AutoscalingClient) ResizeNodePool(id string, targetSize int) error {
client, err := c.GetClient()
func (c *AutoscalingClient) ResizeNodePool(id string, targetSize int32) error {
client, err := c.getClient()
if err != nil {
return err
}
req := client.K8sNodepoolsPut(context.Background(), c.clusterId, id)
req := client.K8sNodepoolsPut(context.Background(), c.cfg.ClusterID, id)
req = req.KubernetesNodePool(resizeRequestBody(targetSize))
_, _, err = client.K8sNodepoolsPutExecute(req)
_, resp, err := client.K8sNodepoolsPutExecute(req)
registerRequest("ResizeNodePool", resp, err)
return err
}

// WaitForNodePoolResize polls the node pool until it is in state ACTIVE.
func (c *AutoscalingClient) WaitForNodePoolResize(id string, size int) error {
klog.V(1).Infof("Waiting for node pool %s to reach target size %d", id, size)
return wait.PollImmediate(c.pollInterval, c.pollTimeout, func() (bool, error) {
nodePool, err := c.GetNodePool(id)
if err != nil {
return false, fmt.Errorf("failed to fetch node pool %s: %w", id, err)
}
state := *nodePool.Metadata.State
klog.V(5).Infof("Polled node pool %s: state=%s", id, state)
return state == K8sStateActive, nil
})
return wait.PollUntilContextTimeout(context.Background(), c.cfg.PollInterval, c.cfg.PollTimeout, true,
func(context.Context) (bool, error) {
nodePool, err := c.GetNodePool(id)
if err != nil {
return false, fmt.Errorf("failed to fetch node pool %s: %w", id, err)
}
state := *nodePool.Metadata.State
klog.V(5).Infof("Polled node pool %s: state=%s", id, state)
return state == ionos.Active, nil
})
}

// ListNodes lists nodes.
func (c *AutoscalingClient) ListNodes(id string) ([]ionos.KubernetesNode, error) {
client, err := c.GetClient()
func (c *AutoscalingClient) ListNodes(nodePoolID string) ([]ionos.KubernetesNode, error) {
client, err := c.getClient()
if err != nil {
return nil, err
}
req := client.K8sNodepoolsNodesGet(context.Background(), c.clusterId, id)
req := client.K8sNodepoolsNodesGet(context.Background(), c.cfg.ClusterID, nodePoolID)
req = req.Depth(1)
nodes, _, err := client.K8sNodepoolsNodesGetExecute(req)
nodes, resp, err := client.K8sNodepoolsNodesGetExecute(req)
registerRequest("ListNodes", resp, err)
if err != nil {
return nil, err
}
return *nodes.Items, nil
}

// DeleteNode starts node deletion.
func (c *AutoscalingClient) DeleteNode(id, nodeId string) error {
client, err := c.GetClient()
func (c *AutoscalingClient) DeleteNode(nodePoolID, nodeID string) error {
client, err := c.getClient()
if err != nil {
return err
}
req := client.K8sNodepoolsNodesDelete(context.Background(), c.clusterId, id, nodeId)
_, err = client.K8sNodepoolsNodesDeleteExecute(req)
req := client.K8sNodepoolsNodesDelete(context.Background(), c.cfg.ClusterID, nodePoolID, nodeID)
resp, err := client.K8sNodepoolsNodesDeleteExecute(req)
registerRequest("DeleteNode", resp, err)
return err
}
29 changes: 13 additions & 16 deletions cluster-autoscaler/cloudprovider/ionoscloud/client_test.go
Original file line number Diff line number Diff line change
@@ -24,27 +24,24 @@ import (
"github.com/stretchr/testify/require"
)

func TestCustomClientProvider(t *testing.T) {
func TestCustomGetClient(t *testing.T) {
tokensPath := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(tokensPath, "..ignoreme"), []byte(`{"invalid"}`), 0644))
require.NoError(t, os.WriteFile(filepath.Join(tokensPath, "..ignoreme"), []byte(`{"invalid"}`), 0o600))

// missing files
provider := customClientProvider{tokensPath, "https://api.ionos.com", "test", true}
_, err := provider.GetClient()
client := NewAutoscalingClient(&Config{
TokensPath: tokensPath,
Endpoint: "https://api.ionos.com",
Insecure: true,
AdditionalHeaders: map[string]string{"Foo": "Bar"},
}, "test")

_, err := client.getClient()
require.Error(t, err)

require.NoError(t, os.WriteFile(filepath.Join(tokensPath, "a"), []byte(`{"tokens":["token1"]}`), 0644))
require.NoError(t, os.WriteFile(filepath.Join(tokensPath, "b"), []byte(`{"tokens":["token2"]}`), 0644))
require.NoError(t, os.WriteFile(filepath.Join(tokensPath, "a"), []byte(`{"tokens":["token1"]}`), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(tokensPath, "b"), []byte(`{"tokens":["token2"]}`), 0o600))

c, err := provider.GetClient()
c, err := client.getClient()
require.NoError(t, err)
require.NotNil(t, c)
}

type fakeClientProvider struct {
client *MockAPIClient
}

func (f fakeClientProvider) GetClient() (APIClient, error) {
return f.client, nil
}

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Loading

0 comments on commit 67895bf

Please sign in to comment.