Add Kafka RestProxy (#123)

* Add Kafka Rest Proxy

Signed-off-by: obaydullahmhs <[email protected]>

go.mod

@@ -32,7 +32,7 @@ require (
 	k8s.io/klog/v2 v2.130.1
 	kmodules.xyz/client-go v0.30.9
 	kmodules.xyz/custom-resources v0.30.0
-	kubedb.dev/apimachinery v0.47.0-rc.1.0.20240717082707-f8438b7e77c7
+	kubedb.dev/apimachinery v0.47.0-rc.1.0.20240726122410-88b60875b260
 	sigs.k8s.io/controller-runtime v0.18.4
 	xorm.io/xorm v1.3.6
 )

go.sum

@@ -793,8 +793,8 @@ kmodules.xyz/monitoring-agent-api v0.29.0 h1:gpFl6OZrlMLb/ySMHdREI9EwGtnJ91oZBn9
 kmodules.xyz/monitoring-agent-api v0.29.0/go.mod h1:iNbvaMTgVFOI5q2LJtGK91j4Dmjv4ZRiRdasGmWLKQI=
 kmodules.xyz/offshoot-api v0.30.0 h1:dq9F93pu4Q8rL9oTcCk+vGGy8vpS7RNt0GSwx7Bvhec=
 kmodules.xyz/offshoot-api v0.30.0/go.mod h1:o9VoA3ImZMDBp3lpLb8+kc2d/KBxioRwCpaKDfLIyDw=
-kubedb.dev/apimachinery v0.47.0-rc.1.0.20240717082707-f8438b7e77c7 h1:HqdKBHKbi5fhHRHstoDggFXpX6v7r+2P8/XevDdHjeg=
-kubedb.dev/apimachinery v0.47.0-rc.1.0.20240717082707-f8438b7e77c7/go.mod h1:Gs/kwdVYmGjJmYmvCUNDmNbbprXqi/gbSj/JrsoM9sE=
+kubedb.dev/apimachinery v0.47.0-rc.1.0.20240726122410-88b60875b260 h1:VE1jdrxBkuwzqVWMafV45sUQHlIBbnO/YrHiOlwHHcA=
+kubedb.dev/apimachinery v0.47.0-rc.1.0.20240726122410-88b60875b260/go.mod h1:Gs/kwdVYmGjJmYmvCUNDmNbbprXqi/gbSj/JrsoM9sE=
 kubeops.dev/petset v0.0.6 h1:0IbvxD9fadZfH+3iMZWzN6ZHsO0vX458JlioamwyPKQ=
 kubeops.dev/petset v0.0.6/go.mod h1:A15vh0r979NsvL65DTIZKWsa/NoX9VapHBAEw1ZsdYI=
 lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=

kafka/restproxy/client.go

@@ -0,0 +1,98 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package restproxy
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/go-resty/resty/v2"
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+)
+
+type Client struct {
+	*resty.Client
+	Config *Config
+}
+
+type Response struct {
+	Code   int
+	Header http.Header
+	Body   io.ReadCloser
+}
+
+type ResponseBody struct {
+	Brokers []int `json:"brokers"`
+}
+
+type Config struct {
+	host             string
+	api              string
+	connectionScheme string
+	transport        *http.Transport
+}
+
+func (cc *Client) GetKafkaBrokerList() (*Response, error) {
+	req := cc.Client.R().SetDoNotParseResponse(true)
+	res, err := req.Get(cc.Config.api)
+	if err != nil {
+		klog.Error(err, "Failed to send http request")
+		return nil, err
+	}
+	response := &Response{
+		Code:   res.StatusCode(),
+		Header: res.Header(),
+		Body:   res.RawBody(),
+	}
+
+	if response.Code != http.StatusOK {
+		return response, fmt.Errorf("failed to get broker list from rest proxy")
+	}
+
+	return response, nil
+}
+
+// IsBrokerAvailableForRequest parse health response in json from server and
+// return overall status of the server
+func (cc *Client) IsBrokerAvailableForRequest(response *Response) (bool, error) {
+	defer func(Body io.ReadCloser) {
+		err := Body.Close()
+		if err != nil {
+			err1 := errors.Wrap(err, "failed to parse response body")
+			if err1 != nil {
+				return
+			}
+			return
+		}
+	}(response.Body)
+
+	var responseBody ResponseBody
+	body, _ := io.ReadAll(response.Body)
+	err := json.Unmarshal(body, &responseBody)
+	if err != nil {
+		return false, errors.Wrap(err, "Failed to parse response body")
+	}
+
+	if len(responseBody.Brokers) == 0 {
+		return false, fmt.Errorf("no brokers found to serve request with rest proxy")
+	}
+
+	return true, nil
+}

kafka/restproxy/kubedb_client_builder.go

@@ -0,0 +1,96 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package restproxy
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/go-resty/resty/v2"
+	kapi "kubedb.dev/apimachinery/apis/kafka/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type KubeDBClientBuilder struct {
+	kc        client.Client
+	restproxy *kapi.RestProxy
+	url       string
+	path      string
+	podName   string
+	ctx       context.Context
+}
+
+func NewKubeDBClientBuilder(kc client.Client, restproxy *kapi.RestProxy) *KubeDBClientBuilder {
+	return &KubeDBClientBuilder{
+		kc:        kc,
+		restproxy: restproxy,
+		url:       GetConnectionURL(restproxy),
+	}
+}
+
+func (o *KubeDBClientBuilder) WithPod(podName string) *KubeDBClientBuilder {
+	o.podName = podName
+	return o
+}
+
+func (o *KubeDBClientBuilder) WithURL(url string) *KubeDBClientBuilder {
+	o.url = url
+	return o
+}
+
+func (o *KubeDBClientBuilder) WithPath(path string) *KubeDBClientBuilder {
+	o.path = path
+	return o
+}
+
+func (o *KubeDBClientBuilder) WithContext(ctx context.Context) *KubeDBClientBuilder {
+	o.ctx = ctx
+	return o
+}
+
+func (o *KubeDBClientBuilder) GetRestProxyClient() (*Client, error) {
+	config := Config{
+		host: o.url,
+		api:  o.path,
+		transport: &http.Transport{
+			IdleConnTimeout: time.Second * 3,
+			DialContext: (&net.Dialer{
+				Timeout: time.Second * 30,
+			}).DialContext,
+		},
+		connectionScheme: o.restproxy.GetConnectionScheme(),
+	}
+
+	newClient := resty.New()
+	newClient.SetTransport(config.transport).SetScheme(config.connectionScheme).SetBaseURL(config.host)
+	newClient.SetHeader("Accept", "application/vnd.kafka.json.v2+json")
+	newClient.SetTimeout(time.Second * 30)
+	newClient.SetDisableWarn(true)
+
+	return &Client{
+		Client: newClient,
+		Config: &config,
+	}, nil
+}
+
+func GetConnectionURL(restproxy *kapi.RestProxy) string {
+	return fmt.Sprintf("%s://%s.%s.svc:%d", restproxy.GetConnectionScheme(), restproxy.ServiceName(), restproxy.Namespace, kapi.RestProxyRESTPort)
+
+}

kafka/schemaregistry/client.go

@@ -18,6 +18,7 @@ package schemaregistry
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 
@@ -92,5 +93,9 @@ func (cc *Client) IsSchemaRegistryHealthy(response *Response) (bool, error) {
 		return false, errors.Wrap(err, "Failed to parse response body")
 	}
 
-	return responseBody.Status == "UP", nil
+	if responseBody.Status != "UP" {
+		return false, fmt.Errorf("schema registry is not healthy")
+	}
+
+	return true, nil
 }

solr/api.go

@@ -1,3 +1,19 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package solr
 
 import (

solr/client.go

@@ -1,3 +1,19 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package solr
 
 import (

solr/kubedb_client_builder.go

@@ -1,3 +1,19 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package solr
 
 import (

solr/solrv8.go

@@ -1,3 +1,19 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package solr
 
 import (

solr/solrv9.go

@@ -1,3 +1,19 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package solr
 
 import (

solr/util.go

@@ -1,3 +1,19 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package solr
 
 import (

vendor/kubedb.dev/apimachinery/apis/kafka/v1alpha1/connect_cluster_webhook.go

@@ -59,7 +59,7 @@ func (k *ConnectCluster) ValidateCreate() (admission.Warnings, error) {
 	if len(allErr) == 0 {
 		return nil, nil
 	}
-	return nil, apierrors.NewInvalid(schema.GroupKind{Group: "kafka.kubedb.com", Kind: "Kafka"}, k.Name, allErr)
+	return nil, apierrors.NewInvalid(schema.GroupKind{Group: "kafka.kubedb.com", Kind: "ConnectCluster"}, k.Name, allErr)
 }
 
 // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
@@ -87,9 +87,9 @@ func (k *ConnectCluster) ValidateDelete() (admission.Warnings, error) {
 
 	var allErr field.ErrorList
 	if k.Spec.DeletionPolicy == dbapi.DeletionPolicyDoNotTerminate {
-		allErr = append(allErr, field.Invalid(field.NewPath("spec").Child("terminationPolicy"),
+		allErr = append(allErr, field.Invalid(field.NewPath("spec").Child("deletionPolicy"),
 			k.Name,
-			"Can not delete as terminationPolicy is set to \"DoNotTerminate\""))
+			"Can not delete as deletionPolicy is set to \"DoNotTerminate\""))
 		return nil, apierrors.NewInvalid(schema.GroupKind{Group: "kafka.kubedb.com", Kind: "ConnectCluster"}, k.Name, allErr)
 	}
 	return nil, nil
@@ -112,7 +112,7 @@ func (k *ConnectCluster) ValidateCreateOrUpdate() field.ErrorList {
 	}
 
 	if k.Spec.DeletionPolicy == dbapi.DeletionPolicyHalt {
-		allErr = append(allErr, field.Invalid(field.NewPath("spec").Child("terminationPolicy"),
+		allErr = append(allErr, field.Invalid(field.NewPath("spec").Child("deletionPolicy"),
 			k.Name,
 			"DeletionPolicyHalt is not supported for ConnectCluster"))
 	}