From 57bcec1e57da12107c11089e8abdb189b75665b3 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 9 Dec 2024 21:14:28 +0300 Subject: [PATCH 01/42] Initial implementation of HTTP to MQTT proxy Signed-off-by: nyagamunene --- go.mod | 9 ++++ go.sum | 12 ++++++ proxy/config.go | 24 +++++++++++ proxy/handler.go | 80 +++++++++++++++++++++++++++++++++++ proxy/http/http.go | 59 ++++++++++++++++++++++++++ proxy/mqtt/mqtt.go | 89 +++++++++++++++++++++++++++++++++++++++ proxy/stream.go | 101 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 374 insertions(+) create mode 100644 proxy/config.go create mode 100644 proxy/handler.go create mode 100644 proxy/http/http.go create mode 100644 proxy/mqtt/mqtt.go create mode 100644 proxy/stream.go diff --git a/go.mod b/go.mod index f7aea4b..de46be4 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( github.com/fatih/color v1.18.0 github.com/go-chi/chi/v5 v5.2.0 github.com/go-kit/kit v0.13.0 + github.com/caarlos0/env/v11 v11.2.2 + github.com/eclipse/paho.mqtt.golang v1.5.0 github.com/google/uuid v1.6.0 github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f github.com/prometheus/client_golang v1.20.5 @@ -54,4 +56,11 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 // indirect google.golang.org/grpc v1.69.0 // indirect google.golang.org/protobuf v1.36.0 // indirect + golang.org/x/sync v0.7.0 + oras.land/oras-go/v2 v2.5.0 +) + +require ( + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 695655c..c4d5bac 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,16 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg= +github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= +github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= +github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= @@ -121,3 +129,7 @@ google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= +oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= diff --git a/proxy/config.go b/proxy/config.go new file mode 100644 index 0000000..6a0722d --- /dev/null +++ b/proxy/config.go @@ -0,0 +1,24 @@ +package proxy + +import "github.com/caarlos0/env/v11" + +type Config struct { + Address string `env:"ADDRESS" envDefault:""` + PathPrefix string `env:"PATH_PREFIX" envDefault:"/"` + Target string `env:"TARGET" envDefault:""` + Root string `env:"ROOT" envDefault:""` + RegURL string `env:"REG_URL" envDefault:""` + UserRepo string `env:"USER_REPO" envDefault:""` + Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` + Username string `env:"USERNAME" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` +} + +func NewConfig(opts env.Options) (Config, error) { + c := Config{} + if err := env.ParseWithOptions(&c, opts); err != nil { + return Config{}, err + } + + return c, nil +} diff --git a/proxy/handler.go b/proxy/handler.go new file mode 100644 index 0000000..546ecee --- /dev/null +++ b/proxy/handler.go @@ -0,0 +1,80 @@ +package proxy + +import ( + "context" + "errors" + "log/slog" +) + +var _ Handler = (*handler)(nil) + +var errSessionMissing = errors.New("session is missing") + +type Session struct { + ID string + Username string + Password []byte +} + +type sessionKey struct{} + +type Handler interface { + Connect(ctx context.Context) error + + Disconnect(ctx context.Context) error + + Publish(ctx context.Context, topic *string, payload *[]byte) error +} + +type handler struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *handler { + return &handler{ + logger: logger, + } +} + +func (h *handler) Connect(ctx context.Context) error { + return h.logAction(ctx, "Connect", nil, nil) +} + +func (h *handler) Publish(ctx context.Context, topic *string, payload *[]byte) error { + return h.logAction(ctx, "Publish", &[]string{*topic}, payload) +} + +func (h *handler) Disconnect(ctx context.Context) error { + return h.logAction(ctx, "Disconnect", nil, nil) +} + +func (h *handler) logAction(ctx context.Context, action string, topics *[]string, payload *[]byte) error { + s, ok := FromContext(ctx) + args := []interface{}{ + slog.Group("session", slog.String("id", s.ID), slog.String("username", s.Username)), + } + + if topics != nil { + args = append(args, slog.Any("topics", *topics)) + } + if payload != nil { + args = append(args, slog.Any("payload", *payload)) + } + + if !ok { + args = append(args, slog.Any("error", errSessionMissing)) + h.logger.Error(action+"() failed to complete", args...) + return errSessionMissing + } + + h.logger.Info(action+"() completed successfully", args...) + + return nil +} + +func FromContext(ctx context.Context) (*Session, bool) { + if s, ok := ctx.Value(sessionKey{}).(*Session); ok && s != nil { + return s, true + } + return nil, false +} diff --git a/proxy/http/http.go b/proxy/http/http.go new file mode 100644 index 0000000..0736814 --- /dev/null +++ b/proxy/http/http.go @@ -0,0 +1,59 @@ +package http + +import ( + "context" + + "github.com/caarlos0/env/v11" + oras "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/oci" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" +) + +const tag = "latest" + +var envPrefix = "ORAS_" + +type Config struct { + Root string `env:"ROOT" envDefault:""` + RegURL string `env:"REG_URL" envDefault:""` + UserRepo string `env:"USER_REPO" envDefault:""` + Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` + Username string `env:"USERNAME" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` +} + +func FetchFromOCI(ctx context.Context) ([]byte, error) { + config := Config{} + if err := env.ParseWithOptions(&config, env.Options{Prefix: envPrefix}); err != nil { + return nil, err + } + store, err := oci.New(config.Root) + if err != nil { + return nil, err + } + + repo, err := remote.NewRepository(config.RegURL + config.UserRepo) + if err != nil { + return nil, err + } + + if config.Authenticate { + repo.Client = &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.NewCache(), + Credential: auth.StaticCredential(config.RegURL, auth.Credential{ + Username: config.Username, + Password: config.Password, + }), + } + } + + manifestDescriptor, err := oras.Copy(ctx, repo, tag, store, tag, oras.DefaultCopyOptions) + if err != nil { + return nil, err + } + + return manifestDescriptor.Data, nil +} diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go new file mode 100644 index 0000000..e69eed2 --- /dev/null +++ b/proxy/mqtt/mqtt.go @@ -0,0 +1,89 @@ +package mqtt + +import ( + "context" + "fmt" + "io" + "log/slog" + "net" + + "github.com/absmach/propeller/proxy" + "golang.org/x/sync/errgroup" +) + +type Proxy struct { + config proxy.Config + handler proxy.Handler + logger *slog.Logger + dialer net.Dialer +} + +func New(config proxy.Config, handler proxy.Handler, logger *slog.Logger) *Proxy { + return &Proxy{ + config: config, + handler: handler, + logger: logger, + } +} + +func (p Proxy) accept(ctx context.Context, l net.Listener) { + for { + select { + case <-ctx.Done(): + return + default: + conn, err := l.Accept() + if err != nil { + p.logger.Warn("Accept error " + err.Error()) + continue + } + p.logger.Info("Accepted new client") + go p.handle(ctx, conn) + } + } +} + +func (p Proxy) handle(ctx context.Context, inbound net.Conn) { + defer p.close(inbound) + outbound, err := p.dialer.Dial("tcp", p.config.Target) + if err != nil { + p.logger.Error("Cannot connect to remote broker " + p.config.Target + " due to: " + err.Error()) + return + } + defer p.close(outbound) + + if err = proxy.Stream(ctx, inbound, outbound, p.handler); err != io.EOF { + p.logger.Warn(err.Error()) + } +} + +// Listen of the server, this will block. +func (p Proxy) Listen(ctx context.Context) error { + l, err := net.Listen("tcp", p.config.Address) + if err != nil { + return err + } + + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { + p.accept(ctx, l) + return nil + }) + + g.Go(func() error { + <-ctx.Done() + return l.Close() + }) + if err := g.Wait(); err != nil { + p.logger.Info(fmt.Sprintf("MQTT proxy server at %s", p.config.Address), slog.String("error", err.Error())) + } + + return nil +} + +func (p Proxy) close(conn net.Conn) { + if err := conn.Close(); err != nil { + p.logger.Warn(fmt.Sprintf("Error closing connection %s", err.Error())) + } +} diff --git a/proxy/stream.go b/proxy/stream.go new file mode 100644 index 0000000..f5476a2 --- /dev/null +++ b/proxy/stream.go @@ -0,0 +1,101 @@ +package proxy + +import ( + "context" + "errors" + "fmt" + "net" + "time" + + orasHTTP "github.com/absmach/propeller/proxy/http" + "github.com/eclipse/paho.mqtt.golang/packets" +) + +func Stream(ctx context.Context, in, out net.Conn, h Handler) error { + errs := make(chan error, 2) + + go streamHTTP(ctx, in, out, h, errs) + go streamMQTT(ctx, out, in, h, errs) + + err := <-errs + + disconnectErr := h.Disconnect(ctx) + + return errors.Join(err, disconnectErr) +} + +func streamHTTP(ctx context.Context, _, w net.Conn, h Handler, errs chan error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + for { + select { + case <-ctx.Done(): + errs <- ctx.Err() + return + default: + data, err := orasHTTP.FetchFromOCI(ctx) + if err != nil { + errs <- err + return + } + + if err = h.Connect(ctx); err != nil { + errs <- err + return + } + + if _, err = w.Write(data); err != nil { + errs <- err + return + } + } + } +} + +func streamMQTT(ctx context.Context, r, w net.Conn, h Handler, errs chan error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + for { + select { + case <-ctx.Done(): + errs <- ctx.Err() + default: + pkt, err := packets.ReadPacket(r) + if err != nil { + errs <- err + return + } + + switch p := pkt.(type) { + case *packets.PublishPacket: + topics := p.TopicName + if err = h.Publish(ctx, &topics, &p.Payload); err != nil { + disconnectPkt := packets.NewControlPacket(packets.Disconnect).(*packets.DisconnectPacket) + if wErr := disconnectPkt.Write(w); wErr != nil { + err = errors.Join(err, wErr) + } + errs <- fmt.Errorf("MQTT publish error: %w", err) + return + } + + case *packets.ConnectPacket: + if err = h.Connect(ctx); err != nil { + errs <- fmt.Errorf("MQTT connection error: %w", err) + return + } + + case *packets.DisconnectPacket: + errs <- h.Disconnect(ctx) + return + } + + if err := pkt.Write(w); err != nil { + errs <- err + return + } + } + + } +} From 3a10e8250762f61d9343d910f32dd65ea37d192d Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Tue, 10 Dec 2024 19:43:44 +0300 Subject: [PATCH 02/42] update mqtt client and http client Signed-off-by: nyagamunene --- go.mod | 2 + go.sum | 4 + proxy/config.go | 26 +---- proxy/http/http.go | 29 ++++-- proxy/mqtt/mqtt.go | 164 +++++++++++++++++++++----------- proxy/{stream.go => service.go} | 34 ++++++- 6 files changed, 171 insertions(+), 88 deletions(-) rename proxy/{stream.go => service.go} (71%) diff --git a/go.mod b/go.mod index de46be4..bf18bf0 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,8 @@ require ( ) require ( + github.com/gorilla/websocket v1.5.3 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect + golang.org/x/net v0.27.0 // indirect ) diff --git a/go.sum b/go.sum index c4d5bac..715cb7a 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,8 @@ github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjO github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -129,6 +131,8 @@ google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= diff --git a/proxy/config.go b/proxy/config.go index 6a0722d..bb287a5 100644 --- a/proxy/config.go +++ b/proxy/config.go @@ -1,24 +1,8 @@ package proxy -import "github.com/caarlos0/env/v11" - -type Config struct { - Address string `env:"ADDRESS" envDefault:""` - PathPrefix string `env:"PATH_PREFIX" envDefault:"/"` - Target string `env:"TARGET" envDefault:""` - Root string `env:"ROOT" envDefault:""` - RegURL string `env:"REG_URL" envDefault:""` - UserRepo string `env:"USER_REPO" envDefault:""` - Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` - Username string `env:"USERNAME" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` -} - -func NewConfig(opts env.Options) (Config, error) { - c := Config{} - if err := env.ParseWithOptions(&c, opts); err != nil { - return Config{}, err - } - - return c, nil +type MQTTProxyConfig struct { + BrokerURL string `json:"broker_url"` + Password string `json:"password"` + PropletID string `json:"proplet_id"` + ChannelID string `json:"channel_id"` } diff --git a/proxy/http/http.go b/proxy/http/http.go index 0736814..9f42118 100644 --- a/proxy/http/http.go +++ b/proxy/http/http.go @@ -16,36 +16,39 @@ const tag = "latest" var envPrefix = "ORAS_" type Config struct { - Root string `env:"ROOT" envDefault:""` - RegURL string `env:"REG_URL" envDefault:""` - UserRepo string `env:"USER_REPO" envDefault:""` + Root string `env:"ROOT" envDefault:"/tmp/oras_oci_folder"` Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` Username string `env:"USERNAME" envDefault:""` Password string `env:"PASSWORD" envDefault:""` } -func FetchFromOCI(ctx context.Context) ([]byte, error) { +func Init() (*Config, error) { config := Config{} if err := env.ParseWithOptions(&config, env.Options{Prefix: envPrefix}); err != nil { return nil, err } - store, err := oci.New(config.Root) + + return &config, nil +} + +func (c *Config) FetchFromReg(ctx context.Context, regURL, filePath string) ([]byte, error) { + store, err := oci.New(c.Root) if err != nil { return nil, err } - repo, err := remote.NewRepository(config.RegURL + config.UserRepo) + repo, err := remote.NewRepository(regURL + filePath) if err != nil { return nil, err } - if config.Authenticate { + if c.Authenticate { repo.Client = &auth.Client{ Client: retry.DefaultClient, Cache: auth.NewCache(), - Credential: auth.StaticCredential(config.RegURL, auth.Credential{ - Username: config.Username, - Password: config.Password, + Credential: auth.StaticCredential(regURL, auth.Credential{ + Username: c.Username, + Password: c.Password, }), } } @@ -55,5 +58,11 @@ func FetchFromOCI(ctx context.Context) ([]byte, error) { return nil, err } + reader, err := store.Fetch(ctx, manifestDescriptor) + if err != nil { + return nil, err + } + defer reader.Close() + return manifestDescriptor.Data, nil } diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index e69eed2..1cf30f6 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -2,88 +2,140 @@ package mqtt import ( "context" + "encoding/json" "fmt" - "io" - "log/slog" - "net" + "log" + "time" "github.com/absmach/propeller/proxy" - "golang.org/x/sync/errgroup" + mqtt "github.com/eclipse/paho.mqtt.golang" ) -type Proxy struct { - config proxy.Config - handler proxy.Handler - logger *slog.Logger - dialer net.Dialer +type RegistryClient struct { + client mqtt.Client + config *proxy.MQTTProxyConfig + messageChan chan string } -func New(config proxy.Config, handler proxy.Handler, logger *slog.Logger) *Proxy { - return &Proxy{ - config: config, - handler: handler, - logger: logger, +func NewMQTTClient(config *proxy.MQTTProxyConfig) (*RegistryClient, error) { + opts := mqtt.NewClientOptions(). + AddBroker(config.BrokerURL). + SetClientID(fmt.Sprintf("Proplet-%s", config.PropletID)). + SetUsername(config.PropletID). + SetPassword(config.Password). + SetCleanSession(true). + SetAutoReconnect(true). + SetConnectTimeout(10 * time.Second). + SetMaxReconnectInterval(1 * time.Minute) + + opts.SetConnectionLostHandler(func(client mqtt.Client, err error) { + log.Printf("MQTT connection lost: %v\n", err) + }) + + opts.SetReconnectingHandler(func(client mqtt.Client, options *mqtt.ClientOptions) { + log.Println("MQTT reconnecting...") + }) + + client := mqtt.NewClient(opts) + + regClient := &RegistryClient{ + client: client, + config: config, + messageChan: make(chan string, 1), + } + + return regClient, nil +} + +func (c *RegistryClient) Connect(ctx context.Context) error { + token := c.client.Connect() + + select { + case <-token.Done(): + if err := token.Error(); err != nil { + return fmt.Errorf("MQTT connection failed: %w", err) + } + case <-ctx.Done(): + return ctx.Err() } + + return c.subscribe(ctx) } -func (p Proxy) accept(ctx context.Context, l net.Listener) { - for { +func (c *RegistryClient) subscribe(ctx context.Context) error { + subTopic := fmt.Sprintf("channels/%s/message/registry/proplet", c.config.ChannelID) + + handler := func(client mqtt.Client, msg mqtt.Message) { + data := msg.Payload() + + var payLoad = struct { + Appname string `json:"app_name"` + }{ + Appname: "", + } + + err := json.Unmarshal(data, &payLoad) + if err != nil { + log.Fatalf("failed unmarshalling") + return + } + packageName := payLoad.Appname select { + case c.messageChan <- packageName: + log.Printf("Received package name: %s", packageName) case <-ctx.Done(): return default: - conn, err := l.Accept() - if err != nil { - p.logger.Warn("Accept error " + err.Error()) - continue - } - p.logger.Info("Accepted new client") - go p.handle(ctx, conn) + log.Println("Package name channel full, dropping message") } } -} -func (p Proxy) handle(ctx context.Context, inbound net.Conn) { - defer p.close(inbound) - outbound, err := p.dialer.Dial("tcp", p.config.Target) - if err != nil { - p.logger.Error("Cannot connect to remote broker " + p.config.Target + " due to: " + err.Error()) - return + token := c.client.Subscribe(subTopic, 1, handler) + if err := token.Error(); err != nil { + return fmt.Errorf("failed to subscribe to %s: %w", subTopic, err) } - defer p.close(outbound) - if err = proxy.Stream(ctx, inbound, outbound, p.handler); err != io.EOF { - p.logger.Warn(err.Error()) - } + return nil } -// Listen of the server, this will block. -func (p Proxy) Listen(ctx context.Context) error { - l, err := net.Listen("tcp", p.config.Address) - if err != nil { - return err - } +// PublishOCIContainer publishes OCI container information +func (c *RegistryClient) PublishOCIContainer(ctx context.Context, containerData []byte) error { + pubTopic := fmt.Sprintf("channels/%s/messages/registry/server", c.config.ChannelID) - g, ctx := errgroup.WithContext(ctx) + token := c.client.Publish(pubTopic, 1, false, containerData) - g.Go(func() error { - p.accept(ctx, l) + select { + case <-token.Done(): + if err := token.Error(); err != nil { + return fmt.Errorf("failed to publish OCI container: %w", err) + } return nil - }) - - g.Go(func() error { - <-ctx.Done() - return l.Close() - }) - if err := g.Wait(); err != nil { - p.logger.Info(fmt.Sprintf("MQTT proxy server at %s", p.config.Address), slog.String("error", err.Error())) - } + case <-ctx.Done(): + return ctx.Err() + } +} - return nil +func (c *RegistryClient) WaitForPackageName(ctx context.Context) (string, error) { + select { + case packageName := <-c.messageChan: + return packageName, nil + case <-ctx.Done(): + return "", ctx.Err() + } } -func (p Proxy) close(conn net.Conn) { - if err := conn.Close(); err != nil { - p.logger.Warn(fmt.Sprintf("Error closing connection %s", err.Error())) +func (c *RegistryClient) Disconnect(ctx context.Context) error { + disconnectChan := make(chan error, 1) + + go func() { + c.client.Disconnect(250) + disconnectChan <- nil + }() + + select { + case err := <-disconnectChan: + return err + case <-ctx.Done(): + return ctx.Err() } } diff --git a/proxy/stream.go b/proxy/service.go similarity index 71% rename from proxy/stream.go rename to proxy/service.go index f5476a2..6664c36 100644 --- a/proxy/stream.go +++ b/proxy/service.go @@ -4,13 +4,37 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "time" orasHTTP "github.com/absmach/propeller/proxy/http" + "github.com/absmach/propeller/proxy/mqtt" "github.com/eclipse/paho.mqtt.golang/packets" ) +type ProxyService struct { + orasconfig orasHTTP.Config + mqttClient mqtt.RegistryClient +} + +func NewService(ctx context.Context, cfg *MQTTProxyConfig, logger *slog.Logger) (*ProxyService, error) { + mqttClient, err := mqtt.NewMQTTClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to initialize MQTT client: %w", err) + } + + config, err := orasHTTP.Init() + if err != nil { + return nil, fmt.Errorf("failed to initialize oras http client: %w", err) + } + + return &ProxyService{ + orasconfig: *config, + mqttClient: mqttClient, + }, nil +} + func Stream(ctx context.Context, in, out net.Conn, h Handler) error { errs := make(chan error, 2) @@ -28,13 +52,21 @@ func streamHTTP(ctx context.Context, _, w net.Conn, h Handler, errs chan error) ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() + // setup to OCI + c, err := orasHTTP.Init() + if err != nil { + errs <- err + return + } + + // check continously for the expected name for { select { case <-ctx.Done(): errs <- ctx.Err() return default: - data, err := orasHTTP.FetchFromOCI(ctx) + data, err := c.FetchFromReg(ctx) if err != nil { errs <- err return From 2c296191bb70ef9c8238dbaaec3f3fe3de8fabea Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 01:08:20 +0300 Subject: [PATCH 03/42] refactor oras http Signed-off-by: nyagamunene --- proxy/{config.go => config/mqtt.go} | 2 +- proxy/http/http.go | 34 ++++---- proxy/mqtt/mqtt.go | 50 +++++------- proxy/service.go | 116 ++++++++++------------------ 4 files changed, 78 insertions(+), 124 deletions(-) rename proxy/{config.go => config/mqtt.go} (92%) diff --git a/proxy/config.go b/proxy/config/mqtt.go similarity index 92% rename from proxy/config.go rename to proxy/config/mqtt.go index bb287a5..8fe213c 100644 --- a/proxy/config.go +++ b/proxy/config/mqtt.go @@ -1,4 +1,4 @@ -package proxy +package config type MQTTProxyConfig struct { BrokerURL string `json:"broker_url"` diff --git a/proxy/http/http.go b/proxy/http/http.go index 9f42118..7e30c8c 100644 --- a/proxy/http/http.go +++ b/proxy/http/http.go @@ -2,10 +2,10 @@ package http import ( "context" + "fmt" + "io" "github.com/caarlos0/env/v11" - oras "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/oci" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" "oras.land/oras-go/v2/registry/remote/retry" @@ -16,7 +16,7 @@ const tag = "latest" var envPrefix = "ORAS_" type Config struct { - Root string `env:"ROOT" envDefault:"/tmp/oras_oci_folder"` + RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` Username string `env:"USERNAME" envDefault:""` Password string `env:"PASSWORD" envDefault:""` @@ -31,38 +31,40 @@ func Init() (*Config, error) { return &config, nil } -func (c *Config) FetchFromReg(ctx context.Context, regURL, filePath string) ([]byte, error) { - store, err := oci.New(c.Root) - if err != nil { - return nil, err - } +func (c *Config) FetchFromReg(ctx context.Context, containerName string) ([]byte, error) { + fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) - repo, err := remote.NewRepository(regURL + filePath) + repo, err := remote.NewRepository(fullPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create repository for %s: %w", containerName, err) } if c.Authenticate { repo.Client = &auth.Client{ Client: retry.DefaultClient, Cache: auth.NewCache(), - Credential: auth.StaticCredential(regURL, auth.Credential{ + Credential: auth.StaticCredential(c.RegistryURL, auth.Credential{ Username: c.Username, Password: c.Password, }), } } - manifestDescriptor, err := oras.Copy(ctx, repo, tag, store, tag, oras.DefaultCopyOptions) + descriptor, err := repo.Resolve(ctx, tag) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to resolve manifest for %s: %w", containerName, err) } - reader, err := store.Fetch(ctx, manifestDescriptor) + reader, err := repo.Fetch(ctx, descriptor) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to fetch blob for %s: %w", containerName, err) } defer reader.Close() - return manifestDescriptor.Data, nil + data, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read blob for %s: %w", containerName, err) + } + + return data, nil } diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index 1cf30f6..fe48665 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -7,17 +7,16 @@ import ( "log" "time" - "github.com/absmach/propeller/proxy" + "github.com/absmach/propeller/proxy/config" mqtt "github.com/eclipse/paho.mqtt.golang" ) type RegistryClient struct { - client mqtt.Client - config *proxy.MQTTProxyConfig - messageChan chan string + client mqtt.Client + config *config.MQTTProxyConfig } -func NewMQTTClient(config *proxy.MQTTProxyConfig) (*RegistryClient, error) { +func NewMQTTClient(config *config.MQTTProxyConfig) (*RegistryClient, error) { opts := mqtt.NewClientOptions(). AddBroker(config.BrokerURL). SetClientID(fmt.Sprintf("Proplet-%s", config.PropletID)). @@ -38,13 +37,10 @@ func NewMQTTClient(config *proxy.MQTTProxyConfig) (*RegistryClient, error) { client := mqtt.NewClient(opts) - regClient := &RegistryClient{ - client: client, - config: config, - messageChan: make(chan string, 1), - } - - return regClient, nil + return &RegistryClient{ + client: client, + config: config, + }, nil } func (c *RegistryClient) Connect(ctx context.Context) error { @@ -59,10 +55,11 @@ func (c *RegistryClient) Connect(ctx context.Context) error { return ctx.Err() } - return c.subscribe(ctx) + return nil } -func (c *RegistryClient) subscribe(ctx context.Context) error { +func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- string) error { + // Subscribe to container requests subTopic := fmt.Sprintf("channels/%s/message/registry/proplet", c.config.ChannelID) handler := func(client mqtt.Client, msg mqtt.Message) { @@ -76,17 +73,17 @@ func (c *RegistryClient) subscribe(ctx context.Context) error { err := json.Unmarshal(data, &payLoad) if err != nil { - log.Fatalf("failed unmarshalling") + log.Printf("failed unmarshalling: %v", err) return } - packageName := payLoad.Appname + select { - case c.messageChan <- packageName: - log.Printf("Received package name: %s", packageName) + case containerChan <- payLoad.Appname: + log.Printf("Received container request: %s", payLoad.Appname) case <-ctx.Done(): return default: - log.Println("Package name channel full, dropping message") + log.Println("Channel full, dropping container request") } } @@ -98,8 +95,8 @@ func (c *RegistryClient) subscribe(ctx context.Context) error { return nil } -// PublishOCIContainer publishes OCI container information -func (c *RegistryClient) PublishOCIContainer(ctx context.Context, containerData []byte) error { +// PublishContainer publishes container data to the server channel +func (c *RegistryClient) PublishContainer(ctx context.Context, containerData []byte) error { pubTopic := fmt.Sprintf("channels/%s/messages/registry/server", c.config.ChannelID) token := c.client.Publish(pubTopic, 1, false, containerData) @@ -107,7 +104,7 @@ func (c *RegistryClient) PublishOCIContainer(ctx context.Context, containerData select { case <-token.Done(): if err := token.Error(); err != nil { - return fmt.Errorf("failed to publish OCI container: %w", err) + return fmt.Errorf("failed to publish container: %w", err) } return nil case <-ctx.Done(): @@ -115,15 +112,6 @@ func (c *RegistryClient) PublishOCIContainer(ctx context.Context, containerData } } -func (c *RegistryClient) WaitForPackageName(ctx context.Context) (string, error) { - select { - case packageName := <-c.messageChan: - return packageName, nil - case <-ctx.Done(): - return "", ctx.Err() - } -} - func (c *RegistryClient) Disconnect(ctx context.Context) error { disconnectChan := make(chan error, 1) diff --git a/proxy/service.go b/proxy/service.go index 6664c36..4931fe1 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -2,23 +2,23 @@ package proxy import ( "context" - "errors" "fmt" "log/slog" - "net" - "time" + "github.com/absmach/propeller/proxy/config" orasHTTP "github.com/absmach/propeller/proxy/http" "github.com/absmach/propeller/proxy/mqtt" - "github.com/eclipse/paho.mqtt.golang/packets" ) type ProxyService struct { - orasconfig orasHTTP.Config - mqttClient mqtt.RegistryClient + orasconfig orasHTTP.Config + mqttClient *mqtt.RegistryClient + logger *slog.Logger + containerChan chan string + dataChan chan []byte } -func NewService(ctx context.Context, cfg *MQTTProxyConfig, logger *slog.Logger) (*ProxyService, error) { +func NewService(ctx context.Context, cfg *config.MQTTProxyConfig, logger *slog.Logger) (*ProxyService, error) { mqttClient, err := mqtt.NewMQTTClient(cfg) if err != nil { return nil, fmt.Errorf("failed to initialize MQTT client: %w", err) @@ -30,104 +30,68 @@ func NewService(ctx context.Context, cfg *MQTTProxyConfig, logger *slog.Logger) } return &ProxyService{ - orasconfig: *config, - mqttClient: mqttClient, + orasconfig: *config, + mqttClient: mqttClient, + logger: logger, + containerChan: make(chan string, 1), + dataChan: make(chan []byte, 1), }, nil } -func Stream(ctx context.Context, in, out net.Conn, h Handler) error { +func (s *ProxyService) Start(ctx context.Context) error { errs := make(chan error, 2) - go streamHTTP(ctx, in, out, h, errs) - go streamMQTT(ctx, out, in, h, errs) + if err := s.mqttClient.Connect(ctx); err != nil { + return fmt.Errorf("failed to connect to MQTT broker: %w", err) + } + defer s.mqttClient.Disconnect(ctx) - err := <-errs + if err := s.mqttClient.Subscribe(ctx, s.containerChan); err != nil { + return fmt.Errorf("failed to subscribe to container requests: %w", err) + } - disconnectErr := h.Disconnect(ctx) + go s.streamHTTP(ctx, errs) + go s.streamMQTT(ctx, errs) - return errors.Join(err, disconnectErr) + return <-errs } -func streamHTTP(ctx context.Context, _, w net.Conn, h Handler, errs chan error) { - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - // setup to OCI - c, err := orasHTTP.Init() - if err != nil { - errs <- err - return - } - - // check continously for the expected name +func (s *ProxyService) streamHTTP(ctx context.Context, errs chan error) { for { select { case <-ctx.Done(): errs <- ctx.Err() return - default: - data, err := c.FetchFromReg(ctx) + case containerName := <-s.containerChan: + data, err := s.orasconfig.FetchFromReg(ctx, containerName) if err != nil { - errs <- err - return - } - - if err = h.Connect(ctx); err != nil { - errs <- err - return + s.logger.Error("failed to fetch container", "container", containerName, "error", err) + continue } - if _, err = w.Write(data); err != nil { - errs <- err + select { + case s.dataChan <- data: + s.logger.Info("sent container data to MQTT stream", "container", containerName) + case <-ctx.Done(): + errs <- ctx.Err() return } } } } -func streamMQTT(ctx context.Context, r, w net.Conn, h Handler, errs chan error) { - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - +func (s *ProxyService) streamMQTT(ctx context.Context, errs chan error) { for { select { case <-ctx.Done(): errs <- ctx.Err() - default: - pkt, err := packets.ReadPacket(r) - if err != nil { - errs <- err - return - } - - switch p := pkt.(type) { - case *packets.PublishPacket: - topics := p.TopicName - if err = h.Publish(ctx, &topics, &p.Payload); err != nil { - disconnectPkt := packets.NewControlPacket(packets.Disconnect).(*packets.DisconnectPacket) - if wErr := disconnectPkt.Write(w); wErr != nil { - err = errors.Join(err, wErr) - } - errs <- fmt.Errorf("MQTT publish error: %w", err) - return - } - - case *packets.ConnectPacket: - if err = h.Connect(ctx); err != nil { - errs <- fmt.Errorf("MQTT connection error: %w", err) - return - } - - case *packets.DisconnectPacket: - errs <- h.Disconnect(ctx) - return - } - - if err := pkt.Write(w); err != nil { - errs <- err - return + return + case data := <-s.dataChan: + if err := s.mqttClient.PublishContainer(ctx, data); err != nil { + s.logger.Error("failed to publish container data", "error", err) + continue } + s.logger.Info("published container data") } - } } From b83e12d137501c3ff2505f7978a7e24d9ac59008 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 14:39:02 +0300 Subject: [PATCH 04/42] add env file and main file Signed-off-by: nyagamunene --- cmd/proxy/.env | 9 +++++ cmd/proxy/main.go | 87 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 cmd/proxy/.env create mode 100644 cmd/proxy/main.go diff --git a/cmd/proxy/.env b/cmd/proxy/.env new file mode 100644 index 0000000..f3dcdc5 --- /dev/null +++ b/cmd/proxy/.env @@ -0,0 +1,9 @@ +MQTT_BROKER_URL=mqtt://localhost:1883 +MQTT_PASSWORD=0e8d1d8d-e3b8-4c20-8873-df200fb56100 +MQTT_PROPLET_ID=7fb3ce2f-271c-4e03-8481-5af7d29f9fd1 +MQTT_CHANNEL_ID=0c5c3658-e069-41d3-b08c-2023f2aad55d + +HTTP_REGISTRY_URL= +HTTP_AUTHENTICATE= +HTTP_USERNAME= +HTTP_PASSWORD= diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go new file mode 100644 index 0000000..350f28b --- /dev/null +++ b/cmd/proxy/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/absmach/propeller/proxy" + "github.com/absmach/propeller/proxy/config" + "github.com/caarlos0/env/v11" + "github.com/joho/godotenv" +) + +const ( + mqttPrefix = "MQTT_" + httpPrefix = "HTTP_" +) + +func main() { + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Set up signal handling + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + // Loading .env file to environment + err := godotenv.Load() + if err != nil { + panic(err) + } + + cfgM, err := config.LoadMQTTConfig(env.Options{Prefix: mqttPrefix}) + if err != nil { + logger.Error("Failed to load MQTT configuration", slog.Any("error", err)) + os.Exit(1) + } + + cfgH, err := config.LoadHTTPConfig(env.Options{Prefix: httpPrefix}) + if err != nil { + logger.Error("Failed to load HTTP configuration", slog.Any("error", err)) + os.Exit(1) + } + + // Create proxy service + service, err := proxy.NewService(ctx, cfgM,cfgH, logger) + if err != nil { + logger.Error("failed to create proxy service", "error", err) + os.Exit(1) + } + + // Start the service + go func() { + if err := start(ctx, service); err != nil { + logger.Error("service error", "error", err) + cancel() + } + }() + + // Wait for signal + <-sigChan + cancel() +} + +func start(ctx context.Context, s *proxy.ProxyService) error { + errs := make(chan error, 2) + + if err := s.MQTTClient().Connect(ctx); err != nil { + return fmt.Errorf("failed to connect to MQTT broker: %w", err) + } + defer s.MQTTClient().Disconnect(ctx) + + if err := s.MQTTClient().Subscribe(ctx, s.ContainerChan()); err != nil { + return fmt.Errorf("failed to subscribe to container requests: %w", err) + } + + go s.StreamHTTP(ctx, errs) + go s.StreamMQTT(ctx, errs) + + return <-errs +} From 3fedc225dec6af13b0ec299181d0271a13902e36 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 14:39:38 +0300 Subject: [PATCH 05/42] add env file and main file Signed-off-by: nyagamunene --- go.mod | 3 +- go.sum | 2 + proxy/{http => config}/http.go | 14 +++--- proxy/config/mqtt.go | 21 +++++++-- proxy/handler.go | 80 ---------------------------------- proxy/mqtt/mqtt.go | 4 +- proxy/service.go | 42 +++++++----------- 7 files changed, 44 insertions(+), 122 deletions(-) rename proxy/{http => config}/http.go (82%) delete mode 100644 proxy/handler.go diff --git a/go.mod b/go.mod index bf18bf0..92dbff5 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.5.0 github.com/google/uuid v1.6.0 github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f + github.com/joho/godotenv v1.5.1 github.com/prometheus/client_golang v1.20.5 github.com/spf13/cobra v1.8.1 github.com/tetratelabs/wazero v1.8.2 @@ -56,7 +57,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 // indirect google.golang.org/grpc v1.69.0 // indirect google.golang.org/protobuf v1.36.0 // indirect - golang.org/x/sync v0.7.0 oras.land/oras-go/v2 v2.5.0 ) @@ -65,4 +65,5 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect golang.org/x/net v0.27.0 // indirect + golang.org/x/sync v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index 715cb7a..b30b771 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= diff --git a/proxy/http/http.go b/proxy/config/http.go similarity index 82% rename from proxy/http/http.go rename to proxy/config/http.go index 7e30c8c..aaf494c 100644 --- a/proxy/http/http.go +++ b/proxy/config/http.go @@ -1,4 +1,4 @@ -package http +package config import ( "context" @@ -13,25 +13,23 @@ import ( const tag = "latest" -var envPrefix = "ORAS_" - -type Config struct { +type HTTPProxyConfig struct { RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` Username string `env:"USERNAME" envDefault:""` Password string `env:"PASSWORD" envDefault:""` } -func Init() (*Config, error) { - config := Config{} - if err := env.ParseWithOptions(&config, env.Options{Prefix: envPrefix}); err != nil { +func LoadHTTPConfig(opts env.Options) (*HTTPProxyConfig, error) { + config := HTTPProxyConfig{} + if err := env.ParseWithOptions(&config, opts); err != nil { return nil, err } return &config, nil } -func (c *Config) FetchFromReg(ctx context.Context, containerName string) ([]byte, error) { +func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]byte, error) { fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) repo, err := remote.NewRepository(fullPath) diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index 8fe213c..66cf910 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -1,8 +1,21 @@ package config +import ( + "github.com/caarlos0/env/v11" +) + type MQTTProxyConfig struct { - BrokerURL string `json:"broker_url"` - Password string `json:"password"` - PropletID string `json:"proplet_id"` - ChannelID string `json:"channel_id"` + BrokerURL string `env:"BROKER_URL" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` + PropletID string `env:"PROPLET_ID" envDefault:""` + ChannelID string `env:"CHANNEL_ID" envDefault:""` +} + +func LoadMQTTConfig(opts env.Options) (*MQTTProxyConfig, error) { + c := MQTTProxyConfig{} + if err := env.ParseWithOptions(&c, opts); err != nil { + return nil, err + } + + return &c, nil } diff --git a/proxy/handler.go b/proxy/handler.go deleted file mode 100644 index 546ecee..0000000 --- a/proxy/handler.go +++ /dev/null @@ -1,80 +0,0 @@ -package proxy - -import ( - "context" - "errors" - "log/slog" -) - -var _ Handler = (*handler)(nil) - -var errSessionMissing = errors.New("session is missing") - -type Session struct { - ID string - Username string - Password []byte -} - -type sessionKey struct{} - -type Handler interface { - Connect(ctx context.Context) error - - Disconnect(ctx context.Context) error - - Publish(ctx context.Context, topic *string, payload *[]byte) error -} - -type handler struct { - logger *slog.Logger -} - -func New(logger *slog.Logger) *handler { - return &handler{ - logger: logger, - } -} - -func (h *handler) Connect(ctx context.Context) error { - return h.logAction(ctx, "Connect", nil, nil) -} - -func (h *handler) Publish(ctx context.Context, topic *string, payload *[]byte) error { - return h.logAction(ctx, "Publish", &[]string{*topic}, payload) -} - -func (h *handler) Disconnect(ctx context.Context) error { - return h.logAction(ctx, "Disconnect", nil, nil) -} - -func (h *handler) logAction(ctx context.Context, action string, topics *[]string, payload *[]byte) error { - s, ok := FromContext(ctx) - args := []interface{}{ - slog.Group("session", slog.String("id", s.ID), slog.String("username", s.Username)), - } - - if topics != nil { - args = append(args, slog.Any("topics", *topics)) - } - if payload != nil { - args = append(args, slog.Any("payload", *payload)) - } - - if !ok { - args = append(args, slog.Any("error", errSessionMissing)) - h.logger.Error(action+"() failed to complete", args...) - return errSessionMissing - } - - h.logger.Info(action+"() completed successfully", args...) - - return nil -} - -func FromContext(ctx context.Context) (*Session, bool) { - if s, ok := ctx.Value(sessionKey{}).(*Session); ok && s != nil { - return s, true - } - return nil, false -} diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index fe48665..e26c7f4 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -17,6 +17,7 @@ type RegistryClient struct { } func NewMQTTClient(config *config.MQTTProxyConfig) (*RegistryClient, error) { + fmt.Printf("config is %+v\n", config) opts := mqtt.NewClientOptions(). AddBroker(config.BrokerURL). SetClientID(fmt.Sprintf("Proplet-%s", config.PropletID)). @@ -59,9 +60,8 @@ func (c *RegistryClient) Connect(ctx context.Context) error { } func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- string) error { - // Subscribe to container requests subTopic := fmt.Sprintf("channels/%s/message/registry/proplet", c.config.ChannelID) - + fmt.Printf("subtopic is %+v\n", subTopic) handler := func(client mqtt.Client, msg mqtt.Message) { data := msg.Payload() diff --git a/proxy/service.go b/proxy/service.go index 4931fe1..a192593 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -6,31 +6,25 @@ import ( "log/slog" "github.com/absmach/propeller/proxy/config" - orasHTTP "github.com/absmach/propeller/proxy/http" "github.com/absmach/propeller/proxy/mqtt" ) type ProxyService struct { - orasconfig orasHTTP.Config + orasconfig *config.HTTPProxyConfig mqttClient *mqtt.RegistryClient logger *slog.Logger containerChan chan string dataChan chan []byte } -func NewService(ctx context.Context, cfg *config.MQTTProxyConfig, logger *slog.Logger) (*ProxyService, error) { - mqttClient, err := mqtt.NewMQTTClient(cfg) +func NewService(ctx context.Context, cfgM *config.MQTTProxyConfig, cfgH *config.HTTPProxyConfig, logger *slog.Logger) (*ProxyService, error) { + mqttClient, err := mqtt.NewMQTTClient(cfgM) if err != nil { return nil, fmt.Errorf("failed to initialize MQTT client: %w", err) } - config, err := orasHTTP.Init() - if err != nil { - return nil, fmt.Errorf("failed to initialize oras http client: %w", err) - } - return &ProxyService{ - orasconfig: *config, + orasconfig: cfgH, mqttClient: mqttClient, logger: logger, containerChan: make(chan string, 1), @@ -38,25 +32,18 @@ func NewService(ctx context.Context, cfg *config.MQTTProxyConfig, logger *slog.L }, nil } -func (s *ProxyService) Start(ctx context.Context) error { - errs := make(chan error, 2) - - if err := s.mqttClient.Connect(ctx); err != nil { - return fmt.Errorf("failed to connect to MQTT broker: %w", err) - } - defer s.mqttClient.Disconnect(ctx) - - if err := s.mqttClient.Subscribe(ctx, s.containerChan); err != nil { - return fmt.Errorf("failed to subscribe to container requests: %w", err) - } - - go s.streamHTTP(ctx, errs) - go s.streamMQTT(ctx, errs) +// MQTTClient returns the MQTT client +func (s *ProxyService) MQTTClient() *mqtt.RegistryClient { + return s.mqttClient +} - return <-errs +// ContainerChan returns the container channel +func (s *ProxyService) ContainerChan() chan string { + return s.containerChan } -func (s *ProxyService) streamHTTP(ctx context.Context, errs chan error) { +// StreamHTTP handles the HTTP stream processing +func (s *ProxyService) StreamHTTP(ctx context.Context, errs chan error) { for { select { case <-ctx.Done(): @@ -80,7 +67,8 @@ func (s *ProxyService) streamHTTP(ctx context.Context, errs chan error) { } } -func (s *ProxyService) streamMQTT(ctx context.Context, errs chan error) { +// StreamMQTT handles the MQTT stream processing +func (s *ProxyService) StreamMQTT(ctx context.Context, errs chan error) { for { select { case <-ctx.Done(): From 0401587151091c796ef257611b603b055ed8074b Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 14:45:17 +0300 Subject: [PATCH 06/42] update go.mod and go.sum file Signed-off-by: nyagamunene --- go.mod | 6 +----- go.sum | 24 ++++++++---------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 92dbff5..9fb788d 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,11 @@ require ( github.com/0x6flab/namegenerator v1.4.0 github.com/absmach/magistrala v0.15.1 github.com/caarlos0/env/v11 v11.3.0 + github.com/caarlos0/env/v11 v11.2.2 github.com/eclipse/paho.mqtt.golang v1.5.0 github.com/fatih/color v1.18.0 github.com/go-chi/chi/v5 v5.2.0 github.com/go-kit/kit v0.13.0 - github.com/caarlos0/env/v11 v11.2.2 - github.com/eclipse/paho.mqtt.golang v1.5.0 github.com/google/uuid v1.6.0 github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f github.com/joho/godotenv v1.5.1 @@ -61,9 +60,6 @@ require ( ) require ( - github.com/gorilla/websocket v1.5.3 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index b30b771..b237335 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3 github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= github.com/caarlos0/env/v11 v11.3.0 h1:CVTN6W6+twFC1jHKUwsw9eOTEiFpzyJOSA2AyHa8uvw= github.com/caarlos0/env/v11 v11.3.0/go.mod h1:Q5lYHeOsgY20CCV/R+b50Jwg2MnjySid7+3FUBz2BJw= +github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg= +github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -40,28 +42,18 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg= -github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= -github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= -github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8= github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -73,6 +65,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= @@ -133,9 +129,5 @@ google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= From e2ac8c5f23ee50b1c7f5c22aa17b4bfbad6a5f7a Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 15:06:06 +0300 Subject: [PATCH 07/42] fix failing linter Signed-off-by: nyagamunene --- cmd/proxy/main.go | 9 ++------- proxy/mqtt/mqtt.go | 27 +++++++++------------------ proxy/service.go | 4 ---- 3 files changed, 11 insertions(+), 29 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 350f28b..40e221f 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -24,13 +24,11 @@ func main() { ctx, cancel := context.WithCancel(ctx) defer cancel() - // Set up signal handling sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - // Loading .env file to environment err := godotenv.Load() if err != nil { panic(err) @@ -44,18 +42,16 @@ func main() { cfgH, err := config.LoadHTTPConfig(env.Options{Prefix: httpPrefix}) if err != nil { - logger.Error("Failed to load HTTP configuration", slog.Any("error", err)) + logger.Error("Failed to load HTTP configuration", slog.Any("error", err)) os.Exit(1) } - // Create proxy service - service, err := proxy.NewService(ctx, cfgM,cfgH, logger) + service, err := proxy.NewService(ctx, cfgM, cfgH, logger) if err != nil { logger.Error("failed to create proxy service", "error", err) os.Exit(1) } - // Start the service go func() { if err := start(ctx, service); err != nil { logger.Error("service error", "error", err) @@ -63,7 +59,6 @@ func main() { } }() - // Wait for signal <-sigChan cancel() } diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index e26c7f4..4c84e0e 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -16,13 +16,12 @@ type RegistryClient struct { config *config.MQTTProxyConfig } -func NewMQTTClient(config *config.MQTTProxyConfig) (*RegistryClient, error) { - fmt.Printf("config is %+v\n", config) +func NewMQTTClient(cfg *config.MQTTProxyConfig) (*RegistryClient, error) { opts := mqtt.NewClientOptions(). - AddBroker(config.BrokerURL). - SetClientID(fmt.Sprintf("Proplet-%s", config.PropletID)). - SetUsername(config.PropletID). - SetPassword(config.Password). + AddBroker(cfg.BrokerURL). + SetClientID(fmt.Sprintf("Proplet-%s", cfg.PropletID)). + SetUsername(cfg.PropletID). + SetPassword(cfg.Password). SetCleanSession(true). SetAutoReconnect(true). SetConnectTimeout(10 * time.Second). @@ -40,7 +39,7 @@ func NewMQTTClient(config *config.MQTTProxyConfig) (*RegistryClient, error) { return &RegistryClient{ client: client, - config: config, + config: cfg, }, nil } @@ -61,7 +60,6 @@ func (c *RegistryClient) Connect(ctx context.Context) error { func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- string) error { subTopic := fmt.Sprintf("channels/%s/message/registry/proplet", c.config.ChannelID) - fmt.Printf("subtopic is %+v\n", subTopic) handler := func(client mqtt.Client, msg mqtt.Message) { data := msg.Payload() @@ -95,7 +93,6 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str return nil } -// PublishContainer publishes container data to the server channel func (c *RegistryClient) PublishContainer(ctx context.Context, containerData []byte) error { pubTopic := fmt.Sprintf("channels/%s/messages/registry/server", c.config.ChannelID) @@ -113,17 +110,11 @@ func (c *RegistryClient) PublishContainer(ctx context.Context, containerData []b } func (c *RegistryClient) Disconnect(ctx context.Context) error { - disconnectChan := make(chan error, 1) - - go func() { - c.client.Disconnect(250) - disconnectChan <- nil - }() - select { - case err := <-disconnectChan: - return err case <-ctx.Done(): return ctx.Err() + default: + c.client.Disconnect(250) + return nil } } diff --git a/proxy/service.go b/proxy/service.go index a192593..8e14a2f 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -32,17 +32,14 @@ func NewService(ctx context.Context, cfgM *config.MQTTProxyConfig, cfgH *config. }, nil } -// MQTTClient returns the MQTT client func (s *ProxyService) MQTTClient() *mqtt.RegistryClient { return s.mqttClient } -// ContainerChan returns the container channel func (s *ProxyService) ContainerChan() chan string { return s.containerChan } -// StreamHTTP handles the HTTP stream processing func (s *ProxyService) StreamHTTP(ctx context.Context, errs chan error) { for { select { @@ -67,7 +64,6 @@ func (s *ProxyService) StreamHTTP(ctx context.Context, errs chan error) { } } -// StreamMQTT handles the MQTT stream processing func (s *ProxyService) StreamMQTT(ctx context.Context, errs chan error) { for { select { From e9755d25acfd4adbe9b014c8b2e66b92349a0035 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 15:11:08 +0300 Subject: [PATCH 08/42] fix tag align Signed-off-by: nyagamunene --- proxy/config/http.go | 8 ++++---- proxy/config/mqtt.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index aaf494c..a085c6f 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -14,10 +14,10 @@ import ( const tag = "latest" type HTTPProxyConfig struct { - RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` - Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` - Username string `env:"USERNAME" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` + RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` + Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` + Username string `env:"USERNAME" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` } func LoadHTTPConfig(opts env.Options) (*HTTPProxyConfig, error) { diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index 66cf910..c25e24e 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -5,10 +5,10 @@ import ( ) type MQTTProxyConfig struct { - BrokerURL string `env:"BROKER_URL" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` - PropletID string `env:"PROPLET_ID" envDefault:""` - ChannelID string `env:"CHANNEL_ID" envDefault:""` + BrokerURL string `json:"broker_url" env:"BROKER_URL"` + Password string `json:"password" env:"PASSWORD"` + PropletID string `json:"proplet_id" env:"PROPLET_ID"` + ChannelID string `json:"channel_id" env:"CHANNEL_ID"` } func LoadMQTTConfig(opts env.Options) (*MQTTProxyConfig, error) { From 16a8e476edf41b03ef90ab87692c23b337b75799 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 15:24:25 +0300 Subject: [PATCH 09/42] fix failing linter Signed-off-by: nyagamunene --- cmd/proxy/main.go | 3 ++- proxy/config/mqtt.go | 8 ++++---- proxy/mqtt/mqtt.go | 16 +++++++++++++--- proxy/service.go | 5 +++++ 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 40e221f..507495d 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -17,6 +17,7 @@ import ( const ( mqttPrefix = "MQTT_" httpPrefix = "HTTP_" + chanSize = 2 ) func main() { @@ -64,7 +65,7 @@ func main() { } func start(ctx context.Context, s *proxy.ProxyService) error { - errs := make(chan error, 2) + errs := make(chan error, chanSize) if err := s.MQTTClient().Connect(ctx); err != nil { return fmt.Errorf("failed to connect to MQTT broker: %w", err) diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index c25e24e..b22bf2f 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -5,10 +5,10 @@ import ( ) type MQTTProxyConfig struct { - BrokerURL string `json:"broker_url" env:"BROKER_URL"` - Password string `json:"password" env:"PASSWORD"` - PropletID string `json:"proplet_id" env:"PROPLET_ID"` - ChannelID string `json:"channel_id" env:"CHANNEL_ID"` + BrokerURL string `env:"BROKER_URL" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` + PropletID string `env:"PROPLET_ID" envDefault:""` + ChannelID string `env:"CHANNEL_ID" envDefault:""` } func LoadMQTTConfig(opts env.Options) (*MQTTProxyConfig, error) { diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index 4c84e0e..19cf4e7 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -11,6 +11,12 @@ import ( mqtt "github.com/eclipse/paho.mqtt.golang" ) +const ( + connTimeout = 10 + reconnTimeout = 1 + disconnTimeout = 250 +) + type RegistryClient struct { client mqtt.Client config *config.MQTTProxyConfig @@ -24,8 +30,8 @@ func NewMQTTClient(cfg *config.MQTTProxyConfig) (*RegistryClient, error) { SetPassword(cfg.Password). SetCleanSession(true). SetAutoReconnect(true). - SetConnectTimeout(10 * time.Second). - SetMaxReconnectInterval(1 * time.Minute) + SetConnectTimeout(connTimeout * time.Second). + SetMaxReconnectInterval(reconnTimeout * time.Minute) opts.SetConnectionLostHandler(func(client mqtt.Client, err error) { log.Printf("MQTT connection lost: %v\n", err) @@ -72,6 +78,7 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str err := json.Unmarshal(data, &payLoad) if err != nil { log.Printf("failed unmarshalling: %v", err) + return } @@ -103,6 +110,7 @@ func (c *RegistryClient) PublishContainer(ctx context.Context, containerData []b if err := token.Error(); err != nil { return fmt.Errorf("failed to publish container: %w", err) } + return nil case <-ctx.Done(): return ctx.Err() @@ -112,9 +120,11 @@ func (c *RegistryClient) PublishContainer(ctx context.Context, containerData []b func (c *RegistryClient) Disconnect(ctx context.Context) error { select { case <-ctx.Done(): + return ctx.Err() default: - c.client.Disconnect(250) + c.client.Disconnect(disconnTimeout) + return nil } } diff --git a/proxy/service.go b/proxy/service.go index 8e14a2f..f3360e8 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -45,11 +45,13 @@ func (s *ProxyService) StreamHTTP(ctx context.Context, errs chan error) { select { case <-ctx.Done(): errs <- ctx.Err() + return case containerName := <-s.containerChan: data, err := s.orasconfig.FetchFromReg(ctx, containerName) if err != nil { s.logger.Error("failed to fetch container", "container", containerName, "error", err) + continue } @@ -58,6 +60,7 @@ func (s *ProxyService) StreamHTTP(ctx context.Context, errs chan error) { s.logger.Info("sent container data to MQTT stream", "container", containerName) case <-ctx.Done(): errs <- ctx.Err() + return } } @@ -69,10 +72,12 @@ func (s *ProxyService) StreamMQTT(ctx context.Context, errs chan error) { select { case <-ctx.Done(): errs <- ctx.Err() + return case data := <-s.dataChan: if err := s.mqttClient.PublishContainer(ctx, data); err != nil { s.logger.Error("failed to publish container data", "error", err) + continue } s.logger.Info("published container data") From 046a1673d13641bb2e2f937317b85d830b4a22da Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 15:41:43 +0300 Subject: [PATCH 10/42] fix start method Signed-off-by: nyagamunene --- cmd/proxy/main.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 507495d..1faf8fc 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -38,19 +38,22 @@ func main() { cfgM, err := config.LoadMQTTConfig(env.Options{Prefix: mqttPrefix}) if err != nil { logger.Error("Failed to load MQTT configuration", slog.Any("error", err)) - os.Exit(1) + + return } cfgH, err := config.LoadHTTPConfig(env.Options{Prefix: httpPrefix}) if err != nil { logger.Error("Failed to load HTTP configuration", slog.Any("error", err)) - os.Exit(1) + + return } service, err := proxy.NewService(ctx, cfgM, cfgH, logger) if err != nil { logger.Error("failed to create proxy service", "error", err) - os.Exit(1) + + return } go func() { @@ -70,7 +73,12 @@ func start(ctx context.Context, s *proxy.ProxyService) error { if err := s.MQTTClient().Connect(ctx); err != nil { return fmt.Errorf("failed to connect to MQTT broker: %w", err) } - defer s.MQTTClient().Disconnect(ctx) + + defer func() { + if err := s.MQTTClient().Disconnect(ctx); err != nil { + slog.Error("failed to disconnect MQTT client", "error", err) + } + }() if err := s.MQTTClient().Subscribe(ctx, s.ContainerChan()); err != nil { return fmt.Errorf("failed to subscribe to container requests: %w", err) From ee096662f8b695531adbe3c2e4ef716a7cda2819 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 15:58:18 +0300 Subject: [PATCH 11/42] remove white spaces Signed-off-by: nyagamunene --- proxy/config/http.go | 8 ++++---- proxy/config/mqtt.go | 8 ++++---- proxy/mqtt/mqtt.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index a085c6f..52e79bb 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -14,10 +14,10 @@ import ( const tag = "latest" type HTTPProxyConfig struct { - RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` - Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` - Username string `env:"USERNAME" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` + RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` + Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` + Username string `env:"USERNAME" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` } func LoadHTTPConfig(opts env.Options) (*HTTPProxyConfig, error) { diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index b22bf2f..95490ed 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -5,10 +5,10 @@ import ( ) type MQTTProxyConfig struct { - BrokerURL string `env:"BROKER_URL" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` - PropletID string `env:"PROPLET_ID" envDefault:""` - ChannelID string `env:"CHANNEL_ID" envDefault:""` + BrokerURL string `env:"BROKER_URL" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` + PropletID string `env:"PROPLET_ID" envDefault:""` + ChannelID string `env:"CHANNEL_ID" envDefault:""` } func LoadMQTTConfig(opts env.Options) (*MQTTProxyConfig, error) { diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index 19cf4e7..7851380 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -25,7 +25,7 @@ type RegistryClient struct { func NewMQTTClient(cfg *config.MQTTProxyConfig) (*RegistryClient, error) { opts := mqtt.NewClientOptions(). AddBroker(cfg.BrokerURL). - SetClientID(fmt.Sprintf("Proplet-%s", cfg.PropletID)). + SetClientID("Proplet-" + cfg.PropletID). SetUsername(cfg.PropletID). SetPassword(cfg.Password). SetCleanSession(true). From b442273be29f022c6edd7937ccb7906cdae75cf4 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 16:00:29 +0300 Subject: [PATCH 12/42] fix failing linter Signed-off-by: nyagamunene --- proxy/config/http.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index 52e79bb..a86726e 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -14,10 +14,10 @@ import ( const tag = "latest" type HTTPProxyConfig struct { - RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` - Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` - Username string `env:"USERNAME" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` + RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` + Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` + Username string `env:"USERNAME" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` } func LoadHTTPConfig(opts env.Options) (*HTTPProxyConfig, error) { From 26a6706709b7ca603357630601f6d104af55ff25 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 11 Dec 2024 16:08:12 +0300 Subject: [PATCH 13/42] fix failing linter Signed-off-by: nyagamunene --- proxy/config/http.go | 4 ++-- proxy/config/mqtt.go | 2 +- proxy/mqtt/mqtt.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index a86726e..aaf494c 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -16,8 +16,8 @@ const tag = "latest" type HTTPProxyConfig struct { RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` - Username string `env:"USERNAME" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` + Username string `env:"USERNAME" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` } func LoadHTTPConfig(opts env.Options) (*HTTPProxyConfig, error) { diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index 95490ed..35843d9 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -6,7 +6,7 @@ import ( type MQTTProxyConfig struct { BrokerURL string `env:"BROKER_URL" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` + Password string `env:"PASSWORD" envDefault:""` PropletID string `env:"PROPLET_ID" envDefault:""` ChannelID string `env:"CHANNEL_ID" envDefault:""` } diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index 7851380..07efbcd 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -69,7 +69,7 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str handler := func(client mqtt.Client, msg mqtt.Message) { data := msg.Payload() - var payLoad = struct { + payLoad := struct { Appname string `json:"app_name"` }{ Appname: "", From 48bf08fa336a6cf0014b505b6e0bddab9d81832f Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 13 Dec 2024 00:20:43 +0300 Subject: [PATCH 14/42] address comments and change how data is sent Signed-off-by: nyagamunene --- cmd/proxy/.env | 8 ++++---- cmd/proxy/main.go | 29 +++++++++-------------------- proxy/config/http.go | 35 ++++++++++++++++++++++++++++++++--- proxy/mqtt/mqtt.go | 20 ++++++++++---------- proxy/service.go | 43 +++++++++++++++++++++++++++---------------- 5 files changed, 82 insertions(+), 53 deletions(-) diff --git a/cmd/proxy/.env b/cmd/proxy/.env index f3dcdc5..88bbb1a 100644 --- a/cmd/proxy/.env +++ b/cmd/proxy/.env @@ -1,7 +1,7 @@ -MQTT_BROKER_URL=mqtt://localhost:1883 -MQTT_PASSWORD=0e8d1d8d-e3b8-4c20-8873-df200fb56100 -MQTT_PROPLET_ID=7fb3ce2f-271c-4e03-8481-5af7d29f9fd1 -MQTT_CHANNEL_ID=0c5c3658-e069-41d3-b08c-2023f2aad55d +MQTT_REGISTRY_BROKER_URL=mqtt://localhost:1883 +MQTT_REGISTRY_PASSWORD=0e8d1d8d-e3b8-4c20-8873-df200fb56100 +MQTT_REGISTRY_PROPLET_ID=7fb3ce2f-271c-4e03-8481-5af7d29f9fd1 +MQTT_REGISTRY_CHANNEL_ID=0c5c3658-e069-41d3-b08c-2023f2aad55d HTTP_REGISTRY_URL= HTTP_AUTHENTICATE= diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 1faf8fc..911390d 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -5,13 +5,12 @@ import ( "fmt" "log/slog" "os" - "os/signal" - "syscall" "github.com/absmach/propeller/proxy" "github.com/absmach/propeller/proxy/config" "github.com/caarlos0/env/v11" "github.com/joho/godotenv" + "golang.org/x/sync/errgroup" ) const ( @@ -21,50 +20,40 @@ const ( ) func main() { - ctx := context.Background() - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + g, ctx := errgroup.WithContext(context.Background()) logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + slog.SetDefault(logger) err := godotenv.Load() if err != nil { panic(err) } - cfgM, err := config.LoadMQTTConfig(env.Options{Prefix: mqttPrefix}) + mqttCfg, err := config.LoadMQTTConfig(env.Options{Prefix: mqttPrefix}) if err != nil { logger.Error("Failed to load MQTT configuration", slog.Any("error", err)) return } - cfgH, err := config.LoadHTTPConfig(env.Options{Prefix: httpPrefix}) + httpCfg, err := config.LoadHTTPConfig(env.Options{Prefix: httpPrefix}) if err != nil { logger.Error("Failed to load HTTP configuration", slog.Any("error", err)) return } - service, err := proxy.NewService(ctx, cfgM, cfgH, logger) + service, err := proxy.NewService(ctx, mqttCfg, httpCfg, logger) if err != nil { logger.Error("failed to create proxy service", "error", err) return } - go func() { - if err := start(ctx, service); err != nil { - logger.Error("service error", "error", err) - cancel() - } - }() - - <-sigChan - cancel() + g.Go(func() error { + return start(ctx, service) + }) } func start(ctx context.Context, s *proxy.ProxyService) error { diff --git a/proxy/config/http.go b/proxy/config/http.go index aaf494c..8b990b6 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -11,7 +11,17 @@ import ( "oras.land/oras-go/v2/registry/remote/retry" ) -const tag = "latest" +const ( + tag = "latest" + chunkSize = 1024 * 1024 +) + +type ChunkPayload struct { + AppName string `json:"app_name"` + ChunkIdx int `json:"chunk_idx"` + TotalChunks int `json:"total_chunks"` + Data []byte `json:"data"` +} type HTTPProxyConfig struct { RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` @@ -29,7 +39,7 @@ func LoadHTTPConfig(opts env.Options) (*HTTPProxyConfig, error) { return &config, nil } -func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]byte, error) { +func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]ChunkPayload, error) { fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) repo, err := remote.NewRepository(fullPath) @@ -64,5 +74,24 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string return nil, fmt.Errorf("failed to read blob for %s: %w", containerName, err) } - return data, nil + totalChunks := (len(data) + chunkSize - 1) / chunkSize + + chunks := make([]ChunkPayload, 0, totalChunks) + for i := 0; i < totalChunks; i++ { + start := i * chunkSize + end := start + chunkSize + if end > len(data) { + end = len(data) + } + + chunk := ChunkPayload{ + AppName: containerName, + ChunkIdx: i, + TotalChunks: totalChunks, + Data: data[start:end], + } + chunks = append(chunks, chunk) + } + + return chunks, nil } diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index 07efbcd..9af17d1 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -15,6 +15,8 @@ const ( connTimeout = 10 reconnTimeout = 1 disconnTimeout = 250 + pubTopic = "channels/%s/messages/registry/server" + subTopic = "channels/%s/message/registry/proplet" ) type RegistryClient struct { @@ -65,7 +67,6 @@ func (c *RegistryClient) Connect(ctx context.Context) error { } func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- string) error { - subTopic := fmt.Sprintf("channels/%s/message/registry/proplet", c.config.ChannelID) handler := func(client mqtt.Client, msg mqtt.Message) { data := msg.Payload() @@ -78,7 +79,6 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str err := json.Unmarshal(data, &payLoad) if err != nil { log.Printf("failed unmarshalling: %v", err) - return } @@ -92,7 +92,7 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str } } - token := c.client.Subscribe(subTopic, 1, handler) + token := c.client.Subscribe(fmt.Sprintf(subTopic, c.config.ChannelID), 1, handler) if err := token.Error(); err != nil { return fmt.Errorf("failed to subscribe to %s: %w", subTopic, err) } @@ -100,17 +100,19 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str return nil } -func (c *RegistryClient) PublishContainer(ctx context.Context, containerData []byte) error { - pubTopic := fmt.Sprintf("channels/%s/messages/registry/server", c.config.ChannelID) +func (c *RegistryClient) PublishContainer(ctx context.Context, chunk config.ChunkPayload) error { + data, err := json.Marshal(chunk) + if err != nil { + return fmt.Errorf("failed to marshal chunk payload: %w", err) + } - token := c.client.Publish(pubTopic, 1, false, containerData) + token := c.client.Publish(fmt.Sprintf(pubTopic, c.config.ChannelID), 1, false, data) select { case <-token.Done(): if err := token.Error(); err != nil { - return fmt.Errorf("failed to publish container: %w", err) + return fmt.Errorf("failed to publish container chunk: %w", err) } - return nil case <-ctx.Done(): return ctx.Err() @@ -120,11 +122,9 @@ func (c *RegistryClient) PublishContainer(ctx context.Context, containerData []b func (c *RegistryClient) Disconnect(ctx context.Context) error { select { case <-ctx.Done(): - return ctx.Err() default: c.client.Disconnect(disconnTimeout) - return nil } } diff --git a/proxy/service.go b/proxy/service.go index f3360e8..40fd1b9 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -14,21 +14,21 @@ type ProxyService struct { mqttClient *mqtt.RegistryClient logger *slog.Logger containerChan chan string - dataChan chan []byte + dataChan chan config.ChunkPayload } -func NewService(ctx context.Context, cfgM *config.MQTTProxyConfig, cfgH *config.HTTPProxyConfig, logger *slog.Logger) (*ProxyService, error) { - mqttClient, err := mqtt.NewMQTTClient(cfgM) +func NewService(ctx context.Context, mqttCfg *config.MQTTProxyConfig, httpCfg *config.HTTPProxyConfig, logger *slog.Logger) (*ProxyService, error) { + mqttClient, err := mqtt.NewMQTTClient(mqttCfg) if err != nil { return nil, fmt.Errorf("failed to initialize MQTT client: %w", err) } return &ProxyService{ - orasconfig: cfgH, + orasconfig: httpCfg, mqttClient: mqttClient, logger: logger, containerChan: make(chan string, 1), - dataChan: make(chan []byte, 1), + dataChan: make(chan config.ChunkPayload, 10), // Increased buffer for chunks }, nil } @@ -48,20 +48,26 @@ func (s *ProxyService) StreamHTTP(ctx context.Context, errs chan error) { return case containerName := <-s.containerChan: - data, err := s.orasconfig.FetchFromReg(ctx, containerName) + chunks, err := s.orasconfig.FetchFromReg(ctx, containerName) if err != nil { s.logger.Error("failed to fetch container", "container", containerName, "error", err) continue } - select { - case s.dataChan <- data: - s.logger.Info("sent container data to MQTT stream", "container", containerName) - case <-ctx.Done(): - errs <- ctx.Err() + // Send each chunk through the data channel + for _, chunk := range chunks { + select { + case s.dataChan <- chunk: + s.logger.Info("sent container chunk to MQTT stream", + "container", containerName, + "chunk", chunk.ChunkIdx, + "total", chunk.TotalChunks) + case <-ctx.Done(): + errs <- ctx.Err() - return + return + } } } } @@ -74,13 +80,18 @@ func (s *ProxyService) StreamMQTT(ctx context.Context, errs chan error) { errs <- ctx.Err() return - case data := <-s.dataChan: - if err := s.mqttClient.PublishContainer(ctx, data); err != nil { - s.logger.Error("failed to publish container data", "error", err) + case chunk := <-s.dataChan: + if err := s.mqttClient.PublishContainer(ctx, chunk); err != nil { + s.logger.Error("failed to publish container chunk", + "error", err, + "chunk", chunk.ChunkIdx, + "total", chunk.TotalChunks) continue } - s.logger.Info("published container data") + s.logger.Info("published container chunk", + "chunk", chunk.ChunkIdx, + "total", chunk.TotalChunks) } } } From ebea46b97d49f0f5496e85c756ab25843b42d6ec Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 13 Dec 2024 00:32:50 +0300 Subject: [PATCH 15/42] fix failing linter Signed-off-by: nyagamunene --- cmd/proxy/main.go | 2 +- proxy/mqtt/mqtt.go | 8 ++++++-- proxy/service.go | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 911390d..63b524e 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -14,7 +14,7 @@ import ( ) const ( - mqttPrefix = "MQTT_" + mqttPrefix = "MQTT_REGISTRY_" httpPrefix = "HTTP_" chanSize = 2 ) diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index 9af17d1..5ff9ccd 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -15,8 +15,8 @@ const ( connTimeout = 10 reconnTimeout = 1 disconnTimeout = 250 - pubTopic = "channels/%s/messages/registry/server" - subTopic = "channels/%s/message/registry/proplet" + pubTopic = "channels/%s/messages/registry/server" + subTopic = "channels/%s/message/registry/proplet" ) type RegistryClient struct { @@ -79,6 +79,7 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str err := json.Unmarshal(data, &payLoad) if err != nil { log.Printf("failed unmarshalling: %v", err) + return } @@ -86,6 +87,7 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str case containerChan <- payLoad.Appname: log.Printf("Received container request: %s", payLoad.Appname) case <-ctx.Done(): + return default: log.Println("Channel full, dropping container request") @@ -113,6 +115,7 @@ func (c *RegistryClient) PublishContainer(ctx context.Context, chunk config.Chun if err := token.Error(); err != nil { return fmt.Errorf("failed to publish container chunk: %w", err) } + return nil case <-ctx.Done(): return ctx.Err() @@ -125,6 +128,7 @@ func (c *RegistryClient) Disconnect(ctx context.Context) error { return ctx.Err() default: c.client.Disconnect(disconnTimeout) + return nil } } diff --git a/proxy/service.go b/proxy/service.go index 40fd1b9..2ecd3f4 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -9,6 +9,8 @@ import ( "github.com/absmach/propeller/proxy/mqtt" ) +const chunkBuffer = 10 + type ProxyService struct { orasconfig *config.HTTPProxyConfig mqttClient *mqtt.RegistryClient @@ -28,7 +30,7 @@ func NewService(ctx context.Context, mqttCfg *config.MQTTProxyConfig, httpCfg *c mqttClient: mqttClient, logger: logger, containerChan: make(chan string, 1), - dataChan: make(chan config.ChunkPayload, 10), // Increased buffer for chunks + dataChan: make(chan config.ChunkPayload, chunkBuffer), }, nil } From 2f4621ae7b2d598d6c4779445491a9104e1bf75c Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 13 Dec 2024 00:40:09 +0300 Subject: [PATCH 16/42] fix failing linter Signed-off-by: nyagamunene --- proxy/config/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index 8b990b6..92fe57e 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -77,7 +77,7 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string totalChunks := (len(data) + chunkSize - 1) / chunkSize chunks := make([]ChunkPayload, 0, totalChunks) - for i := 0; i < totalChunks; i++ { + for i := range make([]struct{}, totalChunks) { start := i * chunkSize end := start + chunkSize if end > len(data) { From 7751314310a9fc73740710d7bdad0ad7a61da6d8 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 13 Dec 2024 01:33:19 +0300 Subject: [PATCH 17/42] add logging Signed-off-by: nyagamunene --- cmd/proxy/.env | 12 ++++++------ cmd/proxy/main.go | 39 +++++++++++++++++++++++++++------------ proxy/service.go | 18 +++++++----------- 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/cmd/proxy/.env b/cmd/proxy/.env index 88bbb1a..fc5da20 100644 --- a/cmd/proxy/.env +++ b/cmd/proxy/.env @@ -1,9 +1,9 @@ -MQTT_REGISTRY_BROKER_URL=mqtt://localhost:1883 -MQTT_REGISTRY_PASSWORD=0e8d1d8d-e3b8-4c20-8873-df200fb56100 -MQTT_REGISTRY_PROPLET_ID=7fb3ce2f-271c-4e03-8481-5af7d29f9fd1 -MQTT_REGISTRY_CHANNEL_ID=0c5c3658-e069-41d3-b08c-2023f2aad55d +MQTT_REGISTRY_BROKER_URL=localhost:1883 +MQTT_REGISTRY_PROPLET_ID=test-proplet +MQTT_REGISTRY_CHANNEL_ID=test-channel +MQTT_REGISTRY_PASSWORD= -HTTP_REGISTRY_URL= -HTTP_AUTHENTICATE= +HTTP_REGISTRY_URL=localhost:5000 +HTTP_AUTHENTICATE=false HTTP_USERNAME= HTTP_PASSWORD= diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 63b524e..4ba43d0 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -14,9 +14,9 @@ import ( ) const ( + svcName = "proxy" mqttPrefix = "MQTT_REGISTRY_" httpPrefix = "HTTP_" - chanSize = 2 ) func main() { @@ -25,25 +25,29 @@ func main() { logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) slog.SetDefault(logger) - err := godotenv.Load() + err := godotenv.Load("cmd/proxy/.env") if err != nil { panic(err) } mqttCfg, err := config.LoadMQTTConfig(env.Options{Prefix: mqttPrefix}) if err != nil { - logger.Error("Failed to load MQTT configuration", slog.Any("error", err)) + logger.Error("failed to load MQTT configuration", slog.Any("error", err)) return } + logger.Info("successfully loaded MQTT config") + httpCfg, err := config.LoadHTTPConfig(env.Options{Prefix: httpPrefix}) if err != nil { - logger.Error("Failed to load HTTP configuration", slog.Any("error", err)) + logger.Error("failed to load HTTP configuration", slog.Any("error", err)) return } + logger.Info("successfully loaded HTTP config") + service, err := proxy.NewService(ctx, mqttCfg, httpCfg, logger) if err != nil { logger.Error("failed to create proxy service", "error", err) @@ -51,18 +55,22 @@ func main() { return } - g.Go(func() error { - return start(ctx, service) - }) + logger.Info("starting proxy service") + + if err := start(ctx, g, service); err != nil { + logger.Error(fmt.Sprintf("%s service exited with error: %s", svcName, err)) + } } -func start(ctx context.Context, s *proxy.ProxyService) error { - errs := make(chan error, chanSize) +func start(ctx context.Context, g *errgroup.Group, s *proxy.ProxyService) error { + slog.Info("connecting...") if err := s.MQTTClient().Connect(ctx); err != nil { return fmt.Errorf("failed to connect to MQTT broker: %w", err) } + slog.Info("successfully connected to broker") + defer func() { if err := s.MQTTClient().Disconnect(ctx); err != nil { slog.Error("failed to disconnect MQTT client", "error", err) @@ -73,8 +81,15 @@ func start(ctx context.Context, s *proxy.ProxyService) error { return fmt.Errorf("failed to subscribe to container requests: %w", err) } - go s.StreamHTTP(ctx, errs) - go s.StreamMQTT(ctx, errs) + slog.Info("successfully subscribed to topic") + + g.Go(func() error { + return s.StreamHTTP(ctx) + }) + + g.Go(func() error { + return s.StreamMQTT(ctx) + }) - return <-errs + return g.Wait() } diff --git a/proxy/service.go b/proxy/service.go index 2ecd3f4..4dc57ac 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -25,6 +25,8 @@ func NewService(ctx context.Context, mqttCfg *config.MQTTProxyConfig, httpCfg *c return nil, fmt.Errorf("failed to initialize MQTT client: %w", err) } + logger.Info("successfully initialized MQTT client") + return &ProxyService{ orasconfig: httpCfg, mqttClient: mqttClient, @@ -42,13 +44,11 @@ func (s *ProxyService) ContainerChan() chan string { return s.containerChan } -func (s *ProxyService) StreamHTTP(ctx context.Context, errs chan error) { +func (s *ProxyService) StreamHTTP(ctx context.Context) error { for { select { case <-ctx.Done(): - errs <- ctx.Err() - - return + return ctx.Err() case containerName := <-s.containerChan: chunks, err := s.orasconfig.FetchFromReg(ctx, containerName) if err != nil { @@ -66,22 +66,18 @@ func (s *ProxyService) StreamHTTP(ctx context.Context, errs chan error) { "chunk", chunk.ChunkIdx, "total", chunk.TotalChunks) case <-ctx.Done(): - errs <- ctx.Err() - - return + return ctx.Err() } } } } } -func (s *ProxyService) StreamMQTT(ctx context.Context, errs chan error) { +func (s *ProxyService) StreamMQTT(ctx context.Context) error { for { select { case <-ctx.Done(): - errs <- ctx.Err() - - return + return ctx.Err() case chunk := <-s.dataChan: if err := s.mqttClient.PublishContainer(ctx, chunk); err != nil { s.logger.Error("failed to publish container chunk", From 182f89f6d6237be5a98cfec5bfa5f149649641cc Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 13 Dec 2024 19:36:42 +0300 Subject: [PATCH 18/42] add documentation and debug connection issue Signed-off-by: nyagamunene --- .gitignore | 1 + cmd/proxy/.env | 9 ----- cmd/proxy/main.go | 47 ++++++++++++------------ proxy/README.md | 87 ++++++++++++++++++++++++++++++++++++++++++++ proxy/config/http.go | 10 ----- proxy/config/mqtt.go | 21 ++--------- proxy/mqtt/mqtt.go | 5 ++- 7 files changed, 120 insertions(+), 60 deletions(-) delete mode 100644 cmd/proxy/.env create mode 100644 proxy/README.md diff --git a/.gitignore b/.gitignore index 3ea852f..0e2f099 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # Propellerd Build build +config.toml diff --git a/cmd/proxy/.env b/cmd/proxy/.env deleted file mode 100644 index fc5da20..0000000 --- a/cmd/proxy/.env +++ /dev/null @@ -1,9 +0,0 @@ -MQTT_REGISTRY_BROKER_URL=localhost:1883 -MQTT_REGISTRY_PROPLET_ID=test-proplet -MQTT_REGISTRY_CHANNEL_ID=test-channel -MQTT_REGISTRY_PASSWORD= - -HTTP_REGISTRY_URL=localhost:5000 -HTTP_AUTHENTICATE=false -HTTP_USERNAME= -HTTP_PASSWORD= diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 4ba43d0..0ba1d9e 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -8,8 +8,6 @@ import ( "github.com/absmach/propeller/proxy" "github.com/absmach/propeller/proxy/config" - "github.com/caarlos0/env/v11" - "github.com/joho/godotenv" "golang.org/x/sync/errgroup" ) @@ -19,36 +17,41 @@ const ( httpPrefix = "HTTP_" ) +const ( + BrokerURL = "localhost:1883" + PropletID = "72fd490b-f91f-47dc-aa0b-a65931719ee1" + ChannelID = "cb6cb9ae-ddcf-41ab-8f32-f3e93b3a3be2" + PropletPassword = "3963a940-332e-4a18-aa57-bab4d4124ab0" + + RegistryURL = "docker.io" + Authenticate = true + RegistryUsername = "mrstevenyaga" + RegistryPassword = "Nya@851612" +) + func main() { g, ctx := errgroup.WithContext(context.Background()) logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) slog.SetDefault(logger) - err := godotenv.Load("cmd/proxy/.env") - if err != nil { - panic(err) - } - - mqttCfg, err := config.LoadMQTTConfig(env.Options{Prefix: mqttPrefix}) - if err != nil { - logger.Error("failed to load MQTT configuration", slog.Any("error", err)) - - return + mqttCfg := config.MQTTProxyConfig{ + BrokerURL: BrokerURL, + Password: PropletPassword, + PropletID: PropletID, + ChannelID: ChannelID, } - logger.Info("successfully loaded MQTT config") - - httpCfg, err := config.LoadHTTPConfig(env.Options{Prefix: httpPrefix}) - if err != nil { - logger.Error("failed to load HTTP configuration", slog.Any("error", err)) - - return + httpCfg := config.HTTPProxyConfig{ + RegistryURL: RegistryURL, + Authenticate: Authenticate, + Username: RegistryUsername, + Password: RegistryPassword, } - logger.Info("successfully loaded HTTP config") + logger.Info("successfully initialized MQTT and HTTP config") - service, err := proxy.NewService(ctx, mqttCfg, httpCfg, logger) + service, err := proxy.NewService(ctx, &mqttCfg, &httpCfg, logger) if err != nil { logger.Error("failed to create proxy service", "error", err) @@ -63,8 +66,6 @@ func main() { } func start(ctx context.Context, g *errgroup.Group, s *proxy.ProxyService) error { - slog.Info("connecting...") - if err := s.MQTTClient().Connect(ctx); err != nil { return fmt.Errorf("failed to connect to MQTT broker: %w", err) } diff --git a/proxy/README.md b/proxy/README.md new file mode 100644 index 0000000..c097a5a --- /dev/null +++ b/proxy/README.md @@ -0,0 +1,87 @@ +# Proxy Service + +The Proxy Service acts as a bridge between MQTT and HTTP protocols in the Propeller system. It enables bidirectional communication between MQTT clients and HTTP endpoints, allowing for seamless integration of different protocols. + +## Overview + +The proxy service performs two main functions: +1. Subscribes to MQTT topics and forwards messages to HTTP endpoints +2. Streams data between MQTT and HTTP protocols + +## Configuration + +The service is configured using environment variables. + +### Environment Variables + +#### MQTT Configuration + +| Variable | Description | Default | Required | +|----------|-------------|---------|----------| +| `BrokerURL` | URL of the MQTT broker | `localhost:1883` | Yes | +| `PropletID` | Unique identifier for the proplet | `72fd490b-f91f-47dc-aa0b-a65931719ee1` | Yes | +| `ChannelID` | Channel identifier for MQTT communication | `cb6cb9ae-ddcf-41ab-8f32-f3e93b3a3be2` | Yes | +| `PropletPassword` | Password for MQTT authentication | `3963a940-332e-4a18-aa57-bab4d4124ab0` | Yes | + +#### Registry Configuration + +| Variable | Description | Default | Required | +|----------|-------------|---------|----------| +| `RegistryURL` | URL of the HTTP registry | `localhost:5000` | Yes | +| `Authenticate` | Enable/disable registry authentication | `false` | No | +| `RegistryUsername` | Username for registry authentication | `""` | Only if `Authenticate=true` | +| `RegistryPassword` | Password for registry authentication | `""` | Only if `Authenticate=true` | + +### Example Configuration +```env +# MQTT Configuration +BrokerURL=localhost:1883 +PropletID=72fd490b-f91f-47dc-aa0b-a65931719ee1 +ChannelID=cb6cb9ae-ddcf-41ab-8f32-f3e93b3a3be2 +PropletPassword=3963a940-332e-4a18-aa57-bab4d4124ab0 + +# Registry Configuration +RegistryURL=localhost:5000 +Authenticate=false +RegistryUsername= +RegistryPassword= +``` + +## Running the Service + +The proxy service can be started by running the main.go file: + +```bash +go run cmd/proxy/main.go +``` + +## Service Flow + +1. **Initialization** + - Loads configuration from environment variables + - Sets up logging + - Creates a new proxy service instance + +2. **Connection** + - Establishes connection to the MQTT broker + - Subscribes to configured topics + - Sets up HTTP streaming + +3. **Operation** + - Runs two concurrent streams: + - StreamHTTP: Handles HTTP communication + - StreamMQTT: Handles MQTT communication + - Uses error groups for graceful error handling and shutdown + +4. **Error Handling** + - Implements comprehensive error logging + - Graceful shutdown with proper resource cleanup + - Automatic disconnection from MQTT broker on service termination + +## HTTP Registry Operations + +The HTTP configuration supports: +- Registry operations with optional authentication +- Automatic retry mechanism for failed requests +- Chunked data handling with configurable chunk size (1MB default) +- Static credential caching for authenticated requests diff --git a/proxy/config/http.go b/proxy/config/http.go index 92fe57e..d3da929 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -5,7 +5,6 @@ import ( "fmt" "io" - "github.com/caarlos0/env/v11" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" "oras.land/oras-go/v2/registry/remote/retry" @@ -30,15 +29,6 @@ type HTTPProxyConfig struct { Password string `env:"PASSWORD" envDefault:""` } -func LoadHTTPConfig(opts env.Options) (*HTTPProxyConfig, error) { - config := HTTPProxyConfig{} - if err := env.ParseWithOptions(&config, opts); err != nil { - return nil, err - } - - return &config, nil -} - func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]ChunkPayload, error) { fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index 35843d9..fc148e5 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -1,21 +1,8 @@ package config -import ( - "github.com/caarlos0/env/v11" -) - type MQTTProxyConfig struct { - BrokerURL string `env:"BROKER_URL" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` - PropletID string `env:"PROPLET_ID" envDefault:""` - ChannelID string `env:"CHANNEL_ID" envDefault:""` -} - -func LoadMQTTConfig(opts env.Options) (*MQTTProxyConfig, error) { - c := MQTTProxyConfig{} - if err := env.ParseWithOptions(&c, opts); err != nil { - return nil, err - } - - return &c, nil + BrokerURL string + Password string + PropletID string + ChannelID string } diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index 5ff9ccd..5a1ff96 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -16,7 +16,7 @@ const ( reconnTimeout = 1 disconnTimeout = 250 pubTopic = "channels/%s/messages/registry/server" - subTopic = "channels/%s/message/registry/proplet" + subTopic = "channels/%s/messages/registry/proplet" ) type RegistryClient struct { @@ -94,6 +94,9 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str } } + x := fmt.Sprintf(subTopic, c.config.ChannelID) + log.Println(x) + token := c.client.Subscribe(fmt.Sprintf(subTopic, c.config.ChannelID), 1, handler) if err := token.Error(); err != nil { return fmt.Errorf("failed to subscribe to %s: %w", subTopic, err) From a86c74943c64b709c13bc74d999396e89f7c0237 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 13 Dec 2024 19:42:09 +0300 Subject: [PATCH 19/42] update go mod and go sum file Signed-off-by: nyagamunene --- go.mod | 2 -- go.sum | 4 ---- 2 files changed, 6 deletions(-) diff --git a/go.mod b/go.mod index 9fb788d..57c05c8 100644 --- a/go.mod +++ b/go.mod @@ -6,14 +6,12 @@ require ( github.com/0x6flab/namegenerator v1.4.0 github.com/absmach/magistrala v0.15.1 github.com/caarlos0/env/v11 v11.3.0 - github.com/caarlos0/env/v11 v11.2.2 github.com/eclipse/paho.mqtt.golang v1.5.0 github.com/fatih/color v1.18.0 github.com/go-chi/chi/v5 v5.2.0 github.com/go-kit/kit v0.13.0 github.com/google/uuid v1.6.0 github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f - github.com/joho/godotenv v1.5.1 github.com/prometheus/client_golang v1.20.5 github.com/spf13/cobra v1.8.1 github.com/tetratelabs/wazero v1.8.2 diff --git a/go.sum b/go.sum index b237335..f8bd833 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,6 @@ github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3 github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= github.com/caarlos0/env/v11 v11.3.0 h1:CVTN6W6+twFC1jHKUwsw9eOTEiFpzyJOSA2AyHa8uvw= github.com/caarlos0/env/v11 v11.3.0/go.mod h1:Q5lYHeOsgY20CCV/R+b50Jwg2MnjySid7+3FUBz2BJw= -github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg= -github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -52,8 +50,6 @@ github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfk github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= From 1f4b1a504083087a5b77a8a655d537b7a9d67284 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 11:10:50 +0300 Subject: [PATCH 20/42] remove password Signed-off-by: nyagamunene --- cmd/proxy/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 0ba1d9e..1471b4b 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -24,9 +24,9 @@ const ( PropletPassword = "3963a940-332e-4a18-aa57-bab4d4124ab0" RegistryURL = "docker.io" - Authenticate = true - RegistryUsername = "mrstevenyaga" - RegistryPassword = "Nya@851612" + Authenticate = false + RegistryUsername = "" + RegistryPassword = "" ) func main() { From 0b323e6267ff2fb889927d0775b83c135634c7aa Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 11:13:27 +0300 Subject: [PATCH 21/42] add comments Signed-off-by: nyagamunene --- cmd/proxy/main.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 1471b4b..ea62d07 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -18,11 +18,13 @@ const ( ) const ( + // MQTT configuration settings BrokerURL = "localhost:1883" - PropletID = "72fd490b-f91f-47dc-aa0b-a65931719ee1" - ChannelID = "cb6cb9ae-ddcf-41ab-8f32-f3e93b3a3be2" - PropletPassword = "3963a940-332e-4a18-aa57-bab4d4124ab0" + PropletID = "test_proplet" + ChannelID = "test_channel" + PropletPassword = "" + // HTTP configuration settings RegistryURL = "docker.io" Authenticate = false RegistryUsername = "" From b1d7f3893d4ce99143a1dec84d7132899733e33d Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 11:14:37 +0300 Subject: [PATCH 22/42] fix failing linter Signed-off-by: nyagamunene --- cmd/proxy/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index ea62d07..9bad5e6 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -18,13 +18,13 @@ const ( ) const ( - // MQTT configuration settings + // MQTT configuration settings. BrokerURL = "localhost:1883" PropletID = "test_proplet" ChannelID = "test_channel" PropletPassword = "" - // HTTP configuration settings + // HTTP configuration settings. RegistryURL = "docker.io" Authenticate = false RegistryUsername = "" From df6c46c928dbf2c29efa281bc2a1b9f0e8f22705 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 11:24:56 +0300 Subject: [PATCH 23/42] add validation Signed-off-by: nyagamunene --- cmd/proxy/main.go | 12 ++++++++++++ proxy/config/http.go | 13 +++++++++++++ proxy/config/mqtt.go | 26 ++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 9bad5e6..d5d44ef 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -44,6 +44,12 @@ func main() { ChannelID: ChannelID, } + if err := mqttCfg.Validate(); err != nil { + logger.Error("failed to initialize mqtt config", "error", err) + + return + } + httpCfg := config.HTTPProxyConfig{ RegistryURL: RegistryURL, Authenticate: Authenticate, @@ -51,6 +57,12 @@ func main() { Password: RegistryPassword, } + if err := httpCfg.Validate(); err != nil { + logger.Error("failed to initialize mqtt config", "error", err) + + return + } + logger.Info("successfully initialized MQTT and HTTP config") service, err := proxy.NewService(ctx, &mqttCfg, &httpCfg, logger) diff --git a/proxy/config/http.go b/proxy/config/http.go index d3da929..7ed8282 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -2,8 +2,10 @@ package config import ( "context" + "errors" "fmt" "io" + "net/url" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" @@ -29,6 +31,17 @@ type HTTPProxyConfig struct { Password string `env:"PASSWORD" envDefault:""` } +func (c *HTTPProxyConfig) Validate() error { + if c.RegistryURL == "" { + return errors.New("broker_url is required") + } + if _, err := url.Parse(c.RegistryURL); err != nil { + return fmt.Errorf("broker_url is not a valid URL: %w", err) + } + + return nil +} + func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]ChunkPayload, error) { fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index fc148e5..ff58937 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -1,8 +1,34 @@ package config +import ( + "errors" + "fmt" + "net/url" +) + type MQTTProxyConfig struct { BrokerURL string Password string PropletID string ChannelID string } + +func (c *MQTTProxyConfig) Validate() error { + if c.BrokerURL == "" { + return errors.New("broker_url is required") + } + if _, err := url.Parse(c.BrokerURL); err != nil { + return fmt.Errorf("broker_url is not a valid URL: %w", err) + } + if c.Password == "" { + return errors.New("password is required") + } + if c.PropletID == "" { + return errors.New("proplet_id is required") + } + if c.ChannelID == "" { + return errors.New("channel_id is required") + } + + return nil +} From 73560029eab027a91486faadfa91fbcaf1d8a9b6 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 14:52:06 +0300 Subject: [PATCH 24/42] adjust size of data sent via nats Signed-off-by: nyagamunene --- cmd/proxy/main.go | 8 ++-- proxy/config/http.go | 97 ++++++++++++++++++++++++++++++++++++++------ proxy/mqtt/mqtt.go | 4 ++ 3 files changed, 91 insertions(+), 18 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index d5d44ef..b633843 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -11,11 +11,7 @@ import ( "golang.org/x/sync/errgroup" ) -const ( - svcName = "proxy" - mqttPrefix = "MQTT_REGISTRY_" - httpPrefix = "HTTP_" -) +const svcName = "proxy" const ( // MQTT configuration settings. @@ -29,6 +25,7 @@ const ( Authenticate = false RegistryUsername = "" RegistryPassword = "" + RegistryPAT = "" ) func main() { @@ -55,6 +52,7 @@ func main() { Authenticate: Authenticate, Username: RegistryUsername, Password: RegistryPassword, + Token: RegistryPAT, } if err := httpCfg.Validate(); err != nil { diff --git a/proxy/config/http.go b/proxy/config/http.go index 7ed8282..43b6c27 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -2,11 +2,14 @@ package config import ( "context" + "encoding/json" "errors" "fmt" "io" + "log" "net/url" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" "oras.land/oras-go/v2/registry/remote/retry" @@ -14,7 +17,7 @@ import ( const ( tag = "latest" - chunkSize = 1024 * 1024 + chunkSize = 512000 // 500KB to ensure we're well under NATS limit ) type ChunkPayload struct { @@ -27,6 +30,7 @@ type ChunkPayload struct { type HTTPProxyConfig struct { RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` + Token string `env:"PAT" envDefault:""` Username string `env:"USERNAME" envDefault:""` Password string `env:"PASSWORD" envDefault:""` } @@ -39,6 +43,19 @@ func (c *HTTPProxyConfig) Validate() error { return fmt.Errorf("broker_url is not a valid URL: %w", err) } + if c.Authenticate { + hasToken := c.Token != "" + hasCredentials := c.Username != "" && c.Password != "" + + if !hasToken && !hasCredentials { + return errors.New("either PAT or username/password must be provided when authentication is enabled") + } + + if hasToken && c.Username == "" { + return errors.New("username is required when using PAT authentication") + } + } + return nil } @@ -51,13 +68,24 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string } if c.Authenticate { - repo.Client = &auth.Client{ - Client: retry.DefaultClient, - Cache: auth.NewCache(), - Credential: auth.StaticCredential(c.RegistryURL, auth.Credential{ + var cred auth.Credential + + if c.Username != "" && c.Password != "" { + cred = auth.Credential{ Username: c.Username, Password: c.Password, - }), + } + } else if c.Token != "" { + cred = auth.Credential{ + Username: c.Username, + AccessToken: c.Token, + } + } + + repo.Client = &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.NewCache(), + Credential: auth.StaticCredential(c.RegistryURL, cred), } } @@ -66,32 +94,75 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string return nil, fmt.Errorf("failed to resolve manifest for %s: %w", containerName, err) } + log.Printf("Container %s:", containerName) + log.Printf("- Manifest size: %d bytes", descriptor.Size) + log.Printf("- Media type: %s", descriptor.MediaType) + reader, err := repo.Fetch(ctx, descriptor) if err != nil { - return nil, fmt.Errorf("failed to fetch blob for %s: %w", containerName, err) + return nil, fmt.Errorf("failed to fetch manifest for %s: %w", containerName, err) } defer reader.Close() - data, err := io.ReadAll(reader) + manifestData, err := io.ReadAll(reader) if err != nil { - return nil, fmt.Errorf("failed to read blob for %s: %w", containerName, err) + return nil, fmt.Errorf("failed to read manifest for %s: %w", containerName, err) } - totalChunks := (len(data) + chunkSize - 1) / chunkSize + var manifest ocispec.Manifest + if err := json.Unmarshal(manifestData, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest for %s: %w", containerName, err) + } + + var largestLayer ocispec.Descriptor + var maxSize int64 + for _, layer := range manifest.Layers { + if layer.Size > maxSize { + maxSize = layer.Size + largestLayer = layer + } + } + + if largestLayer.Size == 0 { + return nil, fmt.Errorf("no valid layers found in manifest for %s", containerName) + } + + log.Printf("- Found largest layer: %d bytes (%.2f MB)", largestLayer.Size, float64(largestLayer.Size)/(1024*1024)) + + layerReader, err := repo.Fetch(ctx, largestLayer) + if err != nil { + return nil, fmt.Errorf("failed to fetch layer for %s: %w", containerName, err) + } + defer layerReader.Close() + + data, err := io.ReadAll(layerReader) + if err != nil { + return nil, fmt.Errorf("failed to read layer for %s: %w", containerName, err) + } + + dataSize := len(data) + totalChunks := (dataSize + chunkSize - 1) / chunkSize + + log.Printf("- Total data size: %d bytes (%.2f MB)", dataSize, float64(dataSize)/(1024*1024)) + log.Printf("- Chunk size: %d bytes (500 KB)", chunkSize) + log.Printf("- Total chunks: %d", totalChunks) chunks := make([]ChunkPayload, 0, totalChunks) for i := range make([]struct{}, totalChunks) { start := i * chunkSize end := start + chunkSize - if end > len(data) { - end = len(data) + if end > dataSize { + end = dataSize } + chunkData := data[start:end] + log.Printf("- Chunk %d size: %d bytes", i, len(chunkData)) + chunk := ChunkPayload{ AppName: containerName, ChunkIdx: i, TotalChunks: totalChunks, - Data: data[start:end], + Data: chunkData, } chunks = append(chunks, chunk) } diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index 5a1ff96..6d52419 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -43,6 +43,10 @@ func NewMQTTClient(cfg *config.MQTTProxyConfig) (*RegistryClient, error) { log.Println("MQTT reconnecting...") }) + opts.SetOnConnectHandler(func(client mqtt.Client) { + log.Println("MQTT connection established successfully") + }) + client := mqtt.NewClient(opts) return &RegistryClient{ From b9a4d2007c3662324df6eb655f7dec26b20b9f52 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 14:59:18 +0300 Subject: [PATCH 25/42] add contants Signed-off-by: nyagamunene --- proxy/config/http.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index 43b6c27..f756c69 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -17,7 +17,8 @@ import ( const ( tag = "latest" - chunkSize = 512000 // 500KB to ensure we're well under NATS limit + size = 1024 * 1024 + chunkSize = 512000 ) type ChunkPayload struct { @@ -127,7 +128,7 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string return nil, fmt.Errorf("no valid layers found in manifest for %s", containerName) } - log.Printf("- Found largest layer: %d bytes (%.2f MB)", largestLayer.Size, float64(largestLayer.Size)/(1024*1024)) + log.Printf("Container size: %d bytes (%.2f MB)", largestLayer.Size, float64(largestLayer.Size)/size) layerReader, err := repo.Fetch(ctx, largestLayer) if err != nil { @@ -143,9 +144,9 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string dataSize := len(data) totalChunks := (dataSize + chunkSize - 1) / chunkSize - log.Printf("- Total data size: %d bytes (%.2f MB)", dataSize, float64(dataSize)/(1024*1024)) - log.Printf("- Chunk size: %d bytes (500 KB)", chunkSize) - log.Printf("- Total chunks: %d", totalChunks) + log.Printf("Total data size: %d bytes (%.2f MB)", dataSize, float64(dataSize)/size) + log.Printf("Chunk size: %d bytes (500 KB)", chunkSize) + log.Printf("Total chunks: %d", totalChunks) chunks := make([]ChunkPayload, 0, totalChunks) for i := range make([]struct{}, totalChunks) { @@ -156,7 +157,7 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string } chunkData := data[start:end] - log.Printf("- Chunk %d size: %d bytes", i, len(chunkData)) + log.Printf("Chunk %d size: %d bytes", i, len(chunkData)) chunk := ChunkPayload{ AppName: containerName, From f9aa43e418a768e1bd9e7001c32d2e677b06cc12 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 15:18:41 +0300 Subject: [PATCH 26/42] refactor FetchFromReg Signed-off-by: nyagamunene --- proxy/config/http.go | 109 +++++++++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 41 deletions(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index f756c69..f7bd073 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -60,36 +60,32 @@ func (c *HTTPProxyConfig) Validate() error { return nil } -func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]ChunkPayload, error) { - fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) - - repo, err := remote.NewRepository(fullPath) - if err != nil { - return nil, fmt.Errorf("failed to create repository for %s: %w", containerName, err) +func (c *HTTPProxyConfig) setupAuthentication(repo *remote.Repository) { + if !c.Authenticate { + return } - if c.Authenticate { - var cred auth.Credential - - if c.Username != "" && c.Password != "" { - cred = auth.Credential{ - Username: c.Username, - Password: c.Password, - } - } else if c.Token != "" { - cred = auth.Credential{ - Username: c.Username, - AccessToken: c.Token, - } + var cred auth.Credential + if c.Username != "" && c.Password != "" { + cred = auth.Credential{ + Username: c.Username, + Password: c.Password, } - - repo.Client = &auth.Client{ - Client: retry.DefaultClient, - Cache: auth.NewCache(), - Credential: auth.StaticCredential(c.RegistryURL, cred), + } else if c.Token != "" { + cred = auth.Credential{ + Username: c.Username, + AccessToken: c.Token, } } + repo.Client = &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.NewCache(), + Credential: auth.StaticCredential(c.RegistryURL, cred), + } +} + +func (c *HTTPProxyConfig) fetchManifest(ctx context.Context, repo *remote.Repository, containerName string) (*ocispec.Manifest, error) { descriptor, err := repo.Resolve(ctx, tag) if err != nil { return nil, fmt.Errorf("failed to resolve manifest for %s: %w", containerName, err) @@ -115,8 +111,13 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string return nil, fmt.Errorf("failed to parse manifest for %s: %w", containerName, err) } + return &manifest, nil +} + +func findLargestLayer(manifest *ocispec.Manifest) (ocispec.Descriptor, error) { var largestLayer ocispec.Descriptor var maxSize int64 + for _, layer := range manifest.Layers { if layer.Size > maxSize { maxSize = layer.Size @@ -125,22 +126,13 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string } if largestLayer.Size == 0 { - return nil, fmt.Errorf("no valid layers found in manifest for %s", containerName) + return ocispec.Descriptor{}, errors.New("no valid layers found in manifest") } - log.Printf("Container size: %d bytes (%.2f MB)", largestLayer.Size, float64(largestLayer.Size)/size) - - layerReader, err := repo.Fetch(ctx, largestLayer) - if err != nil { - return nil, fmt.Errorf("failed to fetch layer for %s: %w", containerName, err) - } - defer layerReader.Close() - - data, err := io.ReadAll(layerReader) - if err != nil { - return nil, fmt.Errorf("failed to read layer for %s: %w", containerName, err) - } + return largestLayer, nil +} +func createChunks(data []byte, containerName string) []ChunkPayload { dataSize := len(data) totalChunks := (dataSize + chunkSize - 1) / chunkSize @@ -159,14 +151,49 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string chunkData := data[start:end] log.Printf("Chunk %d size: %d bytes", i, len(chunkData)) - chunk := ChunkPayload{ + chunks = append(chunks, ChunkPayload{ AppName: containerName, ChunkIdx: i, TotalChunks: totalChunks, Data: chunkData, - } - chunks = append(chunks, chunk) + }) + } + + return chunks +} + +func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]ChunkPayload, error) { + fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) + + repo, err := remote.NewRepository(fullPath) + if err != nil { + return nil, fmt.Errorf("failed to create repository for %s: %w", containerName, err) + } + + c.setupAuthentication(repo) + + manifest, err := c.fetchManifest(ctx, repo, containerName) + if err != nil { + return nil, err + } + + largestLayer, err := findLargestLayer(manifest) + if err != nil { + return nil, fmt.Errorf("failed to find layer for %s: %w", containerName, err) + } + + log.Printf("Container size: %d bytes (%.2f MB)", largestLayer.Size, float64(largestLayer.Size)/size) + + layerReader, err := repo.Fetch(ctx, largestLayer) + if err != nil { + return nil, fmt.Errorf("failed to fetch layer for %s: %w", containerName, err) + } + defer layerReader.Close() + + data, err := io.ReadAll(layerReader) + if err != nil { + return nil, fmt.Errorf("failed to read layer for %s: %w", containerName, err) } - return chunks, nil + return createChunks(data, containerName), nil } From 50f4f3276dcb64bd02248a937570bb3aa6707a49 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 15:49:24 +0300 Subject: [PATCH 27/42] add logging after all chunks were sent sucessfully Signed-off-by: nyagamunene --- proxy/config/http.go | 10 +++------- proxy/service.go | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index f7bd073..ea1a221 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -91,10 +91,6 @@ func (c *HTTPProxyConfig) fetchManifest(ctx context.Context, repo *remote.Reposi return nil, fmt.Errorf("failed to resolve manifest for %s: %w", containerName, err) } - log.Printf("Container %s:", containerName) - log.Printf("- Manifest size: %d bytes", descriptor.Size) - log.Printf("- Media type: %s", descriptor.MediaType) - reader, err := repo.Fetch(ctx, descriptor) if err != nil { return nil, fmt.Errorf("failed to fetch manifest for %s: %w", containerName, err) @@ -136,9 +132,9 @@ func createChunks(data []byte, containerName string) []ChunkPayload { dataSize := len(data) totalChunks := (dataSize + chunkSize - 1) / chunkSize - log.Printf("Total data size: %d bytes (%.2f MB)", dataSize, float64(dataSize)/size) - log.Printf("Chunk size: %d bytes (500 KB)", chunkSize) - log.Printf("Total chunks: %d", totalChunks) + // log.Printf("Total data size: %d bytes (%.2f MB)", dataSize, float64(dataSize)/size) + // log.Printf("Chunk size: %d bytes (500 KB)", chunkSize) + // log.Printf("Total chunks: %d", totalChunks) chunks := make([]ChunkPayload, 0, totalChunks) for i := range make([]struct{}, totalChunks) { diff --git a/proxy/service.go b/proxy/service.go index 4dc57ac..a9ec1c1 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -74,6 +74,8 @@ func (s *ProxyService) StreamHTTP(ctx context.Context) error { } func (s *ProxyService) StreamMQTT(ctx context.Context) error { + containerChunks := make(map[string]int) + for { select { case <-ctx.Done(): @@ -87,9 +89,20 @@ func (s *ProxyService) StreamMQTT(ctx context.Context) error { continue } + s.logger.Info("published container chunk", - "chunk", chunk.ChunkIdx, + "chunk_name", chunk.AppName, + "chunk_no", chunk.ChunkIdx, "total", chunk.TotalChunks) + + containerChunks[chunk.AppName]++ + + if containerChunks[chunk.AppName] == chunk.TotalChunks { + s.logger.Info("successfully sent all chunks", + "container", chunk.AppName, + "total_chunks", chunk.TotalChunks) + delete(containerChunks, chunk.AppName) + } } } } From fa04558cfcdeea678b5d44330c102fe69ce9ff7a Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 18:21:08 +0300 Subject: [PATCH 28/42] change chunk_payload type Signed-off-by: nyagamunene --- proxy/config/http.go | 16 +++++----------- proxy/mqtt/mqtt.go | 3 ++- proxy/service.go | 5 +++-- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index ea1a221..98529c2 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -9,6 +9,7 @@ import ( "log" "net/url" + "github.com/absmach/propeller/proplet" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" @@ -21,13 +22,6 @@ const ( chunkSize = 512000 ) -type ChunkPayload struct { - AppName string `json:"app_name"` - ChunkIdx int `json:"chunk_idx"` - TotalChunks int `json:"total_chunks"` - Data []byte `json:"data"` -} - type HTTPProxyConfig struct { RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` @@ -128,7 +122,7 @@ func findLargestLayer(manifest *ocispec.Manifest) (ocispec.Descriptor, error) { return largestLayer, nil } -func createChunks(data []byte, containerName string) []ChunkPayload { +func createChunks(data []byte, containerName string) []proplet.ChunkPayload { dataSize := len(data) totalChunks := (dataSize + chunkSize - 1) / chunkSize @@ -136,7 +130,7 @@ func createChunks(data []byte, containerName string) []ChunkPayload { // log.Printf("Chunk size: %d bytes (500 KB)", chunkSize) // log.Printf("Total chunks: %d", totalChunks) - chunks := make([]ChunkPayload, 0, totalChunks) + chunks := make([]proplet.ChunkPayload, 0, totalChunks) for i := range make([]struct{}, totalChunks) { start := i * chunkSize end := start + chunkSize @@ -147,7 +141,7 @@ func createChunks(data []byte, containerName string) []ChunkPayload { chunkData := data[start:end] log.Printf("Chunk %d size: %d bytes", i, len(chunkData)) - chunks = append(chunks, ChunkPayload{ + chunks = append(chunks, proplet.ChunkPayload{ AppName: containerName, ChunkIdx: i, TotalChunks: totalChunks, @@ -158,7 +152,7 @@ func createChunks(data []byte, containerName string) []ChunkPayload { return chunks } -func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]ChunkPayload, error) { +func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]proplet.ChunkPayload, error) { fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) repo, err := remote.NewRepository(fullPath) diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt/mqtt.go index 6d52419..e1afb0e 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt/mqtt.go @@ -7,6 +7,7 @@ import ( "log" "time" + "github.com/absmach/propeller/proplet" "github.com/absmach/propeller/proxy/config" mqtt "github.com/eclipse/paho.mqtt.golang" ) @@ -109,7 +110,7 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str return nil } -func (c *RegistryClient) PublishContainer(ctx context.Context, chunk config.ChunkPayload) error { +func (c *RegistryClient) PublishContainer(ctx context.Context, chunk proplet.ChunkPayload) error { data, err := json.Marshal(chunk) if err != nil { return fmt.Errorf("failed to marshal chunk payload: %w", err) diff --git a/proxy/service.go b/proxy/service.go index a9ec1c1..a33f90b 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" + "github.com/absmach/propeller/proplet" "github.com/absmach/propeller/proxy/config" "github.com/absmach/propeller/proxy/mqtt" ) @@ -16,7 +17,7 @@ type ProxyService struct { mqttClient *mqtt.RegistryClient logger *slog.Logger containerChan chan string - dataChan chan config.ChunkPayload + dataChan chan proplet.ChunkPayload } func NewService(ctx context.Context, mqttCfg *config.MQTTProxyConfig, httpCfg *config.HTTPProxyConfig, logger *slog.Logger) (*ProxyService, error) { @@ -32,7 +33,7 @@ func NewService(ctx context.Context, mqttCfg *config.MQTTProxyConfig, httpCfg *c mqttClient: mqttClient, logger: logger, containerChan: make(chan string, 1), - dataChan: make(chan config.ChunkPayload, chunkBuffer), + dataChan: make(chan proplet.ChunkPayload, chunkBuffer), }, nil } From 4c718e40f84cae23ffcbe4563756a642f80399f1 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Mon, 16 Dec 2024 18:42:54 +0300 Subject: [PATCH 29/42] update test documentation Signed-off-by: nyagamunene --- test.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test.md b/test.md index d9a98c6..7be33e6 100644 --- a/test.md +++ b/test.md @@ -102,3 +102,23 @@ export PROPLET_THING_ID="" export PROPLET_THING_KEY="" propeller-proplet ``` + + **Testing proxy.** + +Start proxy + +```bash +go run cmd/proxy/main.go +``` + +Subscibe to MQTT channel to download the requested binary + +```bash +mosquitto_sub -i magistrala -u $PROPLET_THING_ID -P $PROPLET_THING_KEY -t channels/$MANAGER_CHANNEL_ID/messages/registry/server -h localhost +``` + +Publish to MQTT channel to request the container to download + +```bash +mosquitto_pub -i magistrala -u $PROPLET_THING_ID -P $PROPLET_THING_KEY -t channels/$MANAGER_CHANNEL_ID/messages/registry/proplet -h localhost -m '{"app_name":"magistrala/users"}' +``` From 27608f04cbee34babb9e34b5b92ba345b9631fdf Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 18 Dec 2024 14:24:40 +0300 Subject: [PATCH 30/42] update env variables Signed-off-by: nyagamunene --- cmd/proxy/main.go | 40 +++++++++++----------------------------- go.mod | 1 + go.sum | 2 ++ proxy/config/http.go | 26 +++++++++++--------------- proxy/config/mqtt.go | 8 ++++---- proxy/service.go | 2 +- 6 files changed, 30 insertions(+), 49 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index b633843..80cbe62 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -3,60 +3,42 @@ package main import ( "context" "fmt" + "log" "log/slog" "os" "github.com/absmach/propeller/proxy" "github.com/absmach/propeller/proxy/config" + "github.com/caarlos0/env/v11" "golang.org/x/sync/errgroup" ) const svcName = "proxy" -const ( - // MQTT configuration settings. - BrokerURL = "localhost:1883" - PropletID = "test_proplet" - ChannelID = "test_channel" - PropletPassword = "" - - // HTTP configuration settings. - RegistryURL = "docker.io" - Authenticate = false - RegistryUsername = "" - RegistryPassword = "" - RegistryPAT = "" -) - func main() { g, ctx := errgroup.WithContext(context.Background()) logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) slog.SetDefault(logger) - mqttCfg := config.MQTTProxyConfig{ - BrokerURL: BrokerURL, - Password: PropletPassword, - PropletID: PropletID, - ChannelID: ChannelID, + mqttCfg := config.MQTTProxyConfig{} + if err := env.Parse(&mqttCfg); err != nil { + log.Fatalf("failed to load mqtt config : %s", err.Error()) } if err := mqttCfg.Validate(); err != nil { - logger.Error("failed to initialize mqtt config", "error", err) + log.Fatalf("failed to validate mqtt config : %s", err.Error()) return } - httpCfg := config.HTTPProxyConfig{ - RegistryURL: RegistryURL, - Authenticate: Authenticate, - Username: RegistryUsername, - Password: RegistryPassword, - Token: RegistryPAT, + httpCfg := config.HTTPProxyConfig{} + if err := env.Parse(&httpCfg); err != nil { + log.Fatalf("failed to load http config : %s", err.Error()) } if err := httpCfg.Validate(); err != nil { - logger.Error("failed to initialize mqtt config", "error", err) + log.Fatalf("failed to validate http config : %s", err.Error()) return } @@ -65,7 +47,7 @@ func main() { service, err := proxy.NewService(ctx, &mqttCfg, &httpCfg, logger) if err != nil { - logger.Error("failed to create proxy service", "error", err) + logger.Error("failed to create proxy service", slog.Any("error", err)) return } diff --git a/go.mod b/go.mod index 57c05c8..4a6578a 100644 --- a/go.mod +++ b/go.mod @@ -58,6 +58,7 @@ require ( ) require ( + github.com/caarlos0/env/v11 v11.3.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index f8bd833..4fefaf4 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3 github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= github.com/caarlos0/env/v11 v11.3.0 h1:CVTN6W6+twFC1jHKUwsw9eOTEiFpzyJOSA2AyHa8uvw= github.com/caarlos0/env/v11 v11.3.0/go.mod h1:Q5lYHeOsgY20CCV/R+b50Jwg2MnjySid7+3FUBz2BJw= +github.com/caarlos0/env/v11 v11.3.0 h1:CVTN6W6+twFC1jHKUwsw9eOTEiFpzyJOSA2AyHa8uvw= +github.com/caarlos0/env/v11 v11.3.0/go.mod h1:Q5lYHeOsgY20CCV/R+b50Jwg2MnjySid7+3FUBz2BJw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/proxy/config/http.go b/proxy/config/http.go index 98529c2..44a16a1 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -17,17 +17,17 @@ import ( ) const ( - tag = "latest" - size = 1024 * 1024 - chunkSize = 512000 + tag = "latest" + size = 1024 * 1024 ) type HTTPProxyConfig struct { - RegistryURL string `env:"REGISTRY_URL" envDefault:"localhost:5000"` - Authenticate bool `env:"AUTHENTICATE" envDefault:"false"` - Token string `env:"PAT" envDefault:""` - Username string `env:"USERNAME" envDefault:""` - Password string `env:"PASSWORD" envDefault:""` + RegistryURL string `env:"PROXY_REGISTRY_URL" envDefault:""` + ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` + Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` + Token string `env:"PROXY_REGISTRY_TOKEN" envDefault:""` + Username string `env:"PROXY_REGISTRY_USERNAME" envDefault:""` + Password string `env:"PROXY_REGISTRY_PASSWORD" envDefault:""` } func (c *HTTPProxyConfig) Validate() error { @@ -122,14 +122,10 @@ func findLargestLayer(manifest *ocispec.Manifest) (ocispec.Descriptor, error) { return largestLayer, nil } -func createChunks(data []byte, containerName string) []proplet.ChunkPayload { +func createChunks(data []byte, containerName string, chunkSize int) []proplet.ChunkPayload { dataSize := len(data) totalChunks := (dataSize + chunkSize - 1) / chunkSize - // log.Printf("Total data size: %d bytes (%.2f MB)", dataSize, float64(dataSize)/size) - // log.Printf("Chunk size: %d bytes (500 KB)", chunkSize) - // log.Printf("Total chunks: %d", totalChunks) - chunks := make([]proplet.ChunkPayload, 0, totalChunks) for i := range make([]struct{}, totalChunks) { start := i * chunkSize @@ -152,7 +148,7 @@ func createChunks(data []byte, containerName string) []proplet.ChunkPayload { return chunks } -func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string) ([]proplet.ChunkPayload, error) { +func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string, chunkSize int) ([]proplet.ChunkPayload, error) { fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) repo, err := remote.NewRepository(fullPath) @@ -185,5 +181,5 @@ func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string return nil, fmt.Errorf("failed to read layer for %s: %w", containerName, err) } - return createChunks(data, containerName), nil + return createChunks(data, containerName, chunkSize), nil } diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index ff58937..145e3be 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -7,10 +7,10 @@ import ( ) type MQTTProxyConfig struct { - BrokerURL string - Password string - PropletID string - ChannelID string + BrokerURL string `env:"PROXY_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` + Password string `env:"PROXY_PROPLET_KEY" envDefault:""` + PropletID string `env:"PROXY_PROPLET_ID" envDefault:""` + ChannelID string `env:"PROXY_CHANNEL_ID" envDefault:""` } func (c *MQTTProxyConfig) Validate() error { diff --git a/proxy/service.go b/proxy/service.go index a33f90b..74d5e47 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -51,7 +51,7 @@ func (s *ProxyService) StreamHTTP(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case containerName := <-s.containerChan: - chunks, err := s.orasconfig.FetchFromReg(ctx, containerName) + chunks, err := s.orasconfig.FetchFromReg(ctx, containerName, s.orasconfig.ChunkSize) if err != nil { s.logger.Error("failed to fetch container", "container", containerName, "error", err) From fb81865228b1b25f6f1c720ff42cde6d457d674c Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Wed, 18 Dec 2024 19:57:09 +0300 Subject: [PATCH 31/42] fix failing linter Signed-off-by: nyagamunene --- proxy/config/http.go | 8 ++++---- proxy/config/mqtt.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/proxy/config/http.go b/proxy/config/http.go index 44a16a1..a17a404 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -22,10 +22,10 @@ const ( ) type HTTPProxyConfig struct { - RegistryURL string `env:"PROXY_REGISTRY_URL" envDefault:""` - ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` - Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` - Token string `env:"PROXY_REGISTRY_TOKEN" envDefault:""` + RegistryURL string `env:"PROXY_REGISTRY_URL" envDefault:""` + ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` + Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` + Token string `env:"PROXY_REGISTRY_TOKEN" envDefault:""` Username string `env:"PROXY_REGISTRY_USERNAME" envDefault:""` Password string `env:"PROXY_REGISTRY_PASSWORD" envDefault:""` } diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index 145e3be..9db9e36 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -8,9 +8,9 @@ import ( type MQTTProxyConfig struct { BrokerURL string `env:"PROXY_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` - Password string `env:"PROXY_PROPLET_KEY" envDefault:""` - PropletID string `env:"PROXY_PROPLET_ID" envDefault:""` - ChannelID string `env:"PROXY_CHANNEL_ID" envDefault:""` + Password string `env:"PROXY_PROPLET_KEY" envDefault:""` + PropletID string `env:"PROXY_PROPLET_ID" envDefault:""` + ChannelID string `env:"PROXY_CHANNEL_ID" envDefault:""` } func (c *MQTTProxyConfig) Validate() error { From c31f6d5b2c209ee2421036a1e0fa0626d047a5e6 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Thu, 19 Dec 2024 12:55:42 +0300 Subject: [PATCH 32/42] update env file Signed-off-by: nyagamunene --- cmd/proxy/main.go | 21 ++++++++------------- proxy/config/http.go | 37 ++++++------------------------------- proxy/config/mqtt.go | 34 ++++------------------------------ 3 files changed, 18 insertions(+), 74 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 80cbe62..0efb456 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -18,7 +18,14 @@ const svcName = "proxy" func main() { g, ctx := errgroup.WithContext(context.Background()) - logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + var level slog.Level + if err := level.UnmarshalText([]byte("info")); err != nil { + log.Fatalf("failed to parse log level: %s", err.Error()) + } + logHandler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: level, + }) + logger := slog.New(logHandler) slog.SetDefault(logger) mqttCfg := config.MQTTProxyConfig{} @@ -26,23 +33,11 @@ func main() { log.Fatalf("failed to load mqtt config : %s", err.Error()) } - if err := mqttCfg.Validate(); err != nil { - log.Fatalf("failed to validate mqtt config : %s", err.Error()) - - return - } - httpCfg := config.HTTPProxyConfig{} if err := env.Parse(&httpCfg); err != nil { log.Fatalf("failed to load http config : %s", err.Error()) } - if err := httpCfg.Validate(); err != nil { - log.Fatalf("failed to validate http config : %s", err.Error()) - - return - } - logger.Info("successfully initialized MQTT and HTTP config") service, err := proxy.NewService(ctx, &mqttCfg, &httpCfg, logger) diff --git a/proxy/config/http.go b/proxy/config/http.go index a17a404..8b9901c 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "log" - "net/url" "github.com/absmach/propeller/proplet" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -22,36 +21,12 @@ const ( ) type HTTPProxyConfig struct { - RegistryURL string `env:"PROXY_REGISTRY_URL" envDefault:""` - ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` - Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` - Token string `env:"PROXY_REGISTRY_TOKEN" envDefault:""` - Username string `env:"PROXY_REGISTRY_USERNAME" envDefault:""` - Password string `env:"PROXY_REGISTRY_PASSWORD" envDefault:""` -} - -func (c *HTTPProxyConfig) Validate() error { - if c.RegistryURL == "" { - return errors.New("broker_url is required") - } - if _, err := url.Parse(c.RegistryURL); err != nil { - return fmt.Errorf("broker_url is not a valid URL: %w", err) - } - - if c.Authenticate { - hasToken := c.Token != "" - hasCredentials := c.Username != "" && c.Password != "" - - if !hasToken && !hasCredentials { - return errors.New("either PAT or username/password must be provided when authentication is enabled") - } - - if hasToken && c.Username == "" { - return errors.New("username is required when using PAT authentication") - } - } - - return nil + RegistryURL string `env:"PROXY_REGISTRY_URL,notEmpty" envDefault:""` + ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` + Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` + Token string `env:"PROXY_REGISTRY_TOKEN" envDefault:""` + Username string `env:"PROXY_REGISTRY_USERNAME" envDefault:""` + Password string `env:"PROXY_REGISTRY_PASSWORD" envDefault:""` } func (c *HTTPProxyConfig) setupAuthentication(repo *remote.Repository) { diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index 9db9e36..30031aa 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -1,34 +1,8 @@ package config -import ( - "errors" - "fmt" - "net/url" -) - type MQTTProxyConfig struct { - BrokerURL string `env:"PROXY_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` - Password string `env:"PROXY_PROPLET_KEY" envDefault:""` - PropletID string `env:"PROXY_PROPLET_ID" envDefault:""` - ChannelID string `env:"PROXY_CHANNEL_ID" envDefault:""` -} - -func (c *MQTTProxyConfig) Validate() error { - if c.BrokerURL == "" { - return errors.New("broker_url is required") - } - if _, err := url.Parse(c.BrokerURL); err != nil { - return fmt.Errorf("broker_url is not a valid URL: %w", err) - } - if c.Password == "" { - return errors.New("password is required") - } - if c.PropletID == "" { - return errors.New("proplet_id is required") - } - if c.ChannelID == "" { - return errors.New("channel_id is required") - } - - return nil + BrokerURL string `env:"PROXY_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` + Password string `env:"PROXY_PROPLET_KEY,notEmpty" envDefault:""` + PropletID string `env:"PROXY_PROPLET_ID,notEmpty" envDefault:""` + ChannelID string `env:"PROXY_CHANNEL_ID,notEmpty" envDefault:""` } From 93cb573559acc8bbbc4e23555e5a36d20349108d Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Thu, 19 Dec 2024 14:55:15 +0300 Subject: [PATCH 33/42] remove proxy read me Signed-off-by: nyagamunene --- proxy/README.md | 87 ------------------------------------------------- 1 file changed, 87 deletions(-) delete mode 100644 proxy/README.md diff --git a/proxy/README.md b/proxy/README.md deleted file mode 100644 index c097a5a..0000000 --- a/proxy/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Proxy Service - -The Proxy Service acts as a bridge between MQTT and HTTP protocols in the Propeller system. It enables bidirectional communication between MQTT clients and HTTP endpoints, allowing for seamless integration of different protocols. - -## Overview - -The proxy service performs two main functions: -1. Subscribes to MQTT topics and forwards messages to HTTP endpoints -2. Streams data between MQTT and HTTP protocols - -## Configuration - -The service is configured using environment variables. - -### Environment Variables - -#### MQTT Configuration - -| Variable | Description | Default | Required | -|----------|-------------|---------|----------| -| `BrokerURL` | URL of the MQTT broker | `localhost:1883` | Yes | -| `PropletID` | Unique identifier for the proplet | `72fd490b-f91f-47dc-aa0b-a65931719ee1` | Yes | -| `ChannelID` | Channel identifier for MQTT communication | `cb6cb9ae-ddcf-41ab-8f32-f3e93b3a3be2` | Yes | -| `PropletPassword` | Password for MQTT authentication | `3963a940-332e-4a18-aa57-bab4d4124ab0` | Yes | - -#### Registry Configuration - -| Variable | Description | Default | Required | -|----------|-------------|---------|----------| -| `RegistryURL` | URL of the HTTP registry | `localhost:5000` | Yes | -| `Authenticate` | Enable/disable registry authentication | `false` | No | -| `RegistryUsername` | Username for registry authentication | `""` | Only if `Authenticate=true` | -| `RegistryPassword` | Password for registry authentication | `""` | Only if `Authenticate=true` | - -### Example Configuration -```env -# MQTT Configuration -BrokerURL=localhost:1883 -PropletID=72fd490b-f91f-47dc-aa0b-a65931719ee1 -ChannelID=cb6cb9ae-ddcf-41ab-8f32-f3e93b3a3be2 -PropletPassword=3963a940-332e-4a18-aa57-bab4d4124ab0 - -# Registry Configuration -RegistryURL=localhost:5000 -Authenticate=false -RegistryUsername= -RegistryPassword= -``` - -## Running the Service - -The proxy service can be started by running the main.go file: - -```bash -go run cmd/proxy/main.go -``` - -## Service Flow - -1. **Initialization** - - Loads configuration from environment variables - - Sets up logging - - Creates a new proxy service instance - -2. **Connection** - - Establishes connection to the MQTT broker - - Subscribes to configured topics - - Sets up HTTP streaming - -3. **Operation** - - Runs two concurrent streams: - - StreamHTTP: Handles HTTP communication - - StreamMQTT: Handles MQTT communication - - Uses error groups for graceful error handling and shutdown - -4. **Error Handling** - - Implements comprehensive error logging - - Graceful shutdown with proper resource cleanup - - Automatic disconnection from MQTT broker on service termination - -## HTTP Registry Operations - -The HTTP configuration supports: -- Registry operations with optional authentication -- Automatic retry mechanism for failed requests -- Chunked data handling with configurable chunk size (1MB default) -- Static credential caching for authenticated requests From 471d22e606407e99fc0dd7444da2351df0cf9fb8 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Thu, 19 Dec 2024 19:15:18 +0300 Subject: [PATCH 34/42] intergrate proplet and manager with proxy Signed-off-by: nyagamunene --- .golangci.yaml | 1 + Makefile | 8 +- cmd/proplet/main.go | 39 +--------- cmd/proxy/main.go | 11 ++- manager/api/requests.go | 4 + proplet/requests.go | 13 ++-- proplet/service.go | 158 +++++++++++----------------------------- proxy/config/http.go | 34 ++++----- proxy/config/mqtt.go | 8 +- task/task.go | 21 +++--- test.md | 19 +++-- 11 files changed, 110 insertions(+), 206 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 49823f9..6bf3b58 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -33,6 +33,7 @@ linters: - err113 - noctx - cyclop + - tagalign linters-settings: gocritic: diff --git a/Makefile b/Makefile index 57467f8..30d9855 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ TIME=$(shell date -u '+%Y-%m-%dT%H:%M:%SZ') VERSION ?= $(shell git describe --abbrev=0 --tags 2>/dev/null || echo 'v0.0.0') COMMIT ?= $(shell git rev-parse HEAD) EXAMPLES = addition long-addition -SERVICES = manager proplet cli +SERVICES = manager proplet cli proxy define compile_service CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) \ @@ -22,9 +22,9 @@ $(SERVICES): install: for file in $(BUILD_DIR)/*; do \ - if [[ ! "$$file" =~ \.wasm$$ ]]; then \ - cp "$$file" $(GOBIN)/propeller-`basename "$$file"`; \ - fi \ + if [ "$$file" = "$${file%%.wasm}" ]; then \ + cp "$$file" "$(GOBIN)/propeller-$$(basename "$$file")"; \ + fi; \ done .PHONY: all $(SERVICES) diff --git a/cmd/proplet/main.go b/cmd/proplet/main.go index c6f5f92..7bb28f2 100644 --- a/cmd/proplet/main.go +++ b/cmd/proplet/main.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "log/slog" - "net/http" "os" "time" @@ -26,8 +25,6 @@ type config struct { MQTTTimeout time.Duration `env:"PROPLET_MQTT_TIMEOUT" envDefault:"30s"` MQTTQoS byte `env:"PROPLET_MQTT_QOS" envDefault:"2"` LivelinessInterval time.Duration `env:"PROPLET_LIVELINESS_INTERVAL" envDefault:"10s"` - RegistryURL string `env:"PROPLET_REGISTRY_URL"` - RegistryToken string `env:"PROPLET_REGISTRY_TOKEN"` RegistryTimeout time.Duration `env:"PROPLET_REGISTRY_TIMEOUT" envDefault:"30s"` ChannelID string `env:"PROPLET_CHANNEL_ID,notEmpty"` ThingID string `env:"PROPLET_THING_ID,notEmpty"` @@ -57,16 +54,6 @@ func main() { logger := slog.New(logHandler) slog.SetDefault(logger) - if cfg.RegistryURL != "" { - if err := checkRegistryConnectivity(ctx, cfg.RegistryURL, cfg.RegistryTimeout); err != nil { - logger.Error("failed to connect to registry URL", slog.String("url", cfg.RegistryURL), slog.Any("error", err)) - - return - } - - logger.Info("successfully connected to registry URL", slog.String("url", cfg.RegistryURL)) - } - mqttPubSub, err := mqtt.NewPubSub(cfg.MQTTAddress, cfg.MQTTQoS, cfg.InstanceID, cfg.ThingID, cfg.ThingKey, cfg.ChannelID, cfg.MQTTTimeout, logger) if err != nil { logger.Error("failed to initialize mqtt client", slog.Any("error", err)) @@ -75,7 +62,7 @@ func main() { } wazero := proplet.NewWazeroRuntime(logger, mqttPubSub, cfg.ChannelID) - service, err := proplet.NewService(ctx, cfg.ChannelID, cfg.ThingID, cfg.ThingKey, cfg.RegistryURL, cfg.RegistryToken, cfg.LivelinessInterval, mqttPubSub, logger, wazero) + service, err := proplet.NewService(ctx, cfg.ChannelID, cfg.ThingID, cfg.ThingKey, cfg.LivelinessInterval, mqttPubSub, logger, wazero) if err != nil { logger.Error("failed to initialize service", slog.Any("error", err)) @@ -96,27 +83,3 @@ func main() { logger.Error(fmt.Sprintf("%s service exited with error: %s", svcName, err)) } } - -func checkRegistryConnectivity(ctx context.Context, registryURL string, registryTimeout time.Duration) error { - ctx, cancel := context.WithTimeout(ctx, registryTimeout) - defer cancel() - - client := http.DefaultClient - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, registryURL, http.NoBody) - if err != nil { - return fmt.Errorf("failed to create HTTP request: %w", err) - } - - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("failed to connect to registry URL: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("registry returned unexpected status: %d", resp.StatusCode) - } - - return nil -} diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 0efb456..7c5c90d 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -13,13 +13,16 @@ import ( "golang.org/x/sync/errgroup" ) -const svcName = "proxy" +const ( + svcName = "proxy" + logLevel = "info" +) func main() { g, ctx := errgroup.WithContext(context.Background()) var level slog.Level - if err := level.UnmarshalText([]byte("info")); err != nil { + if err := level.UnmarshalText([]byte(logLevel)); err != nil { log.Fatalf("failed to parse log level: %s", err.Error()) } logHandler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ @@ -30,12 +33,12 @@ func main() { mqttCfg := config.MQTTProxyConfig{} if err := env.Parse(&mqttCfg); err != nil { - log.Fatalf("failed to load mqtt config : %s", err.Error()) + logger.Error("failed to load mqtt config", slog.Any("error", err)) } httpCfg := config.HTTPProxyConfig{} if err := env.Parse(&httpCfg); err != nil { - log.Fatalf("failed to load http config : %s", err.Error()) + logger.Error("failed to load http config", slog.Any("error", err)) } logger.Info("successfully initialized MQTT and HTTP config") diff --git a/manager/api/requests.go b/manager/api/requests.go index d5f23c5..6d27c7d 100644 --- a/manager/api/requests.go +++ b/manager/api/requests.go @@ -10,6 +10,10 @@ type taskReq struct { } func (t *taskReq) validate() error { + if t.Name == "" { + return apiutil.ErrMissingName + } + return nil } diff --git a/proplet/requests.go b/proplet/requests.go index cbd0c2a..8d42405 100644 --- a/proplet/requests.go +++ b/proplet/requests.go @@ -3,10 +3,11 @@ package proplet import "errors" type startRequest struct { - ID string - FunctionName string - WasmFile []byte - Params []uint64 + ID string + FunctionName string + WasmFile []byte + WasmFileDownloadPath string + Params []uint64 } func (r startRequest) Validate() error { @@ -16,8 +17,8 @@ func (r startRequest) Validate() error { if r.FunctionName == "" { return errors.New("function name is required") } - if r.WasmFile == nil { - return errors.New("wasm file is required") + if r.WasmFile == nil && r.WasmFileDownloadPath == "" { + return errors.New("either wasm file or wasm file download path is required") } return nil diff --git a/proplet/service.go b/proplet/service.go index 3f83bfe..f5428c3 100644 --- a/proplet/service.go +++ b/proplet/service.go @@ -7,7 +7,6 @@ import ( "fmt" "log" "log/slog" - "net/url" "sync" "time" @@ -21,22 +20,19 @@ const ( ) var ( - RegistryAckTopicTemplate = "channels/%s/messages/control/manager/registry" - updateRegistryTopicTemplate = "channels/%s/messages/control/manager/update" - aliveTopicTemplate = "channels/%s/messages/control/proplet/alive" - discoveryTopicTemplate = "channels/%s/messages/control/proplet/create" - startTopicTemplate = "channels/%s/messages/control/manager/start" - stopTopicTemplate = "channels/%s/messages/control/manager/stop" - registryResponseTopic = "channels/%s/messages/registry/server" - fetchRequestTopicTemplate = "channels/%s/messages/registry/proplet" + RegistryAckTopicTemplate = "channels/%s/messages/control/manager/registry" + aliveTopicTemplate = "channels/%s/messages/control/proplet/alive" + discoveryTopicTemplate = "channels/%s/messages/control/proplet/create" + startTopicTemplate = "channels/%s/messages/control/manager/start" + stopTopicTemplate = "channels/%s/messages/control/manager/stop" + registryResponseTopic = "channels/%s/messages/registry/server" + fetchRequestTopicTemplate = "channels/%s/messages/registry/proplet" ) type PropletService struct { channelID string thingID string thingKey string - registryURL string - registryToken string livelinessInterval time.Duration pubsub pkgmqtt.PubSub chunks map[string][][]byte @@ -53,7 +49,7 @@ type ChunkPayload struct { Data []byte `json:"data"` } -func NewService(ctx context.Context, channelID, thingID, thingKey, registryURL, registryToken string, livelinessInterval time.Duration, pubsub pkgmqtt.PubSub, logger *slog.Logger, runtime Runtime) (*PropletService, error) { +func NewService(ctx context.Context, channelID, thingID, thingKey string, livelinessInterval time.Duration, pubsub pkgmqtt.PubSub, logger *slog.Logger, runtime Runtime) (*PropletService, error) { topic := fmt.Sprintf(discoveryTopicTemplate, channelID) payload := map[string]interface{}{ "proplet_id": thingID, @@ -67,8 +63,6 @@ func NewService(ctx context.Context, channelID, thingID, thingKey, registryURL, channelID: channelID, thingID: thingID, thingKey: thingKey, - registryURL: registryURL, - registryToken: registryToken, livelinessInterval: livelinessInterval, pubsub: pubsub, chunks: make(map[string][][]byte), @@ -125,11 +119,6 @@ func (p *PropletService) Run(ctx context.Context, logger *slog.Logger) error { return fmt.Errorf("failed to subscribe to registry topics: %w", err) } - topic = fmt.Sprintf(updateRegistryTopicTemplate, p.channelID) - if err := p.pubsub.Subscribe(ctx, topic, p.registryUpdate(ctx)); err != nil { - return fmt.Errorf("failed to subscribe to update registry topic: %w", err) - } - logger.Info("Proplet service is running.") <-ctx.Done() @@ -149,10 +138,11 @@ func (p *PropletService) handleStartCommand(ctx context.Context) func(topic stri } req := startRequest{ - ID: payload.ID, - FunctionName: payload.Name, - WasmFile: payload.File, - Params: payload.Inputs, + ID: payload.ID, + FunctionName: payload.Name, + WasmFile: payload.File, + WasmFileDownloadPath: payload.DownloadFile, + Params: payload.Inputs, } if err := req.Validate(); err != nil { return err @@ -160,39 +150,44 @@ func (p *PropletService) handleStartCommand(ctx context.Context) func(topic stri p.logger.Info("Received start command", slog.String("app_name", req.FunctionName)) - if err := p.runtime.StartApp(ctx, req.WasmFile, req.ID, req.FunctionName, req.Params...); err != nil { - return err - } - - if p.registryURL != "" { - payload := map[string]interface{}{ - "app_name": req.FunctionName, - } - topic := fmt.Sprintf(fetchRequestTopicTemplate, p.channelID) - if err := p.pubsub.Publish(ctx, topic, payload); err != nil { + if req.WasmFile != nil { + if err := p.runtime.StartApp(ctx, req.WasmFile, req.ID, req.FunctionName, req.Params...); err != nil { return err } - go func() { - p.logger.Info("Waiting for chunks", slog.String("app_name", req.FunctionName)) + return nil + } + + pl := map[string]interface{}{ + "app_name": req.WasmFileDownloadPath, + } + tp := fmt.Sprintf(fetchRequestTopicTemplate, p.channelID) + if err := p.pubsub.Publish(ctx, tp, pl); err != nil { + return err + } - for { - p.chunksMutex.Lock() - metadata, exists := p.chunkMetadata[req.FunctionName] - receivedChunks := len(p.chunks[req.FunctionName]) - p.chunksMutex.Unlock() + go func() { + p.logger.Info("Waiting for chunks", slog.String("app_name", req.WasmFileDownloadPath)) - if exists && receivedChunks == metadata.TotalChunks { - p.logger.Info("All chunks received, deploying app", slog.String("app_name", req.FunctionName)) - go p.deployAndRunApp(ctx, req.FunctionName) + for { + p.chunksMutex.Lock() + metadata, exists := p.chunkMetadata[req.WasmFileDownloadPath] + receivedChunks := len(p.chunks[req.WasmFileDownloadPath]) + p.chunksMutex.Unlock() - break + if exists && receivedChunks == metadata.TotalChunks { + p.logger.Info("All chunks received, deploying app", slog.String("app_name", req.WasmFileDownloadPath)) + wasmBinary := assembleChunks(p.chunks[req.WasmFileDownloadPath]) + if err := p.runtime.StartApp(ctx, wasmBinary, req.ID, req.FunctionName, req.Params...); err != nil { + p.logger.Error("Failed to start app", slog.String("app_name", req.WasmFileDownloadPath), slog.Any("error", err)) } - time.Sleep(pollingInterval) + break } - }() - } + + time.Sleep(pollingInterval) + } + }() return nil } @@ -222,7 +217,7 @@ func (p *PropletService) handleStopCommand(ctx context.Context) func(topic strin } } -func (p *PropletService) handleChunk(ctx context.Context) func(topic string, msg map[string]interface{}) error { +func (p *PropletService) handleChunk(_ context.Context) func(topic string, msg map[string]interface{}) error { return func(topic string, msg map[string]interface{}) error { data, err := json.Marshal(msg) if err != nil { @@ -249,29 +244,10 @@ func (p *PropletService) handleChunk(ctx context.Context) func(topic string, msg log.Printf("Received chunk %d/%d for app '%s'\n", chunk.ChunkIdx+1, chunk.TotalChunks, chunk.AppName) - if len(p.chunks[chunk.AppName]) == p.chunkMetadata[chunk.AppName].TotalChunks { - log.Printf("All chunks received for app '%s'. Deploying...\n", chunk.AppName) - go p.deployAndRunApp(ctx, chunk.AppName) - } - return nil } } -func (p *PropletService) deployAndRunApp(ctx context.Context, appName string) { - log.Printf("Assembling chunks for app '%s'\n", appName) - - p.chunksMutex.Lock() - chunks := p.chunks[appName] - delete(p.chunks, appName) - p.chunksMutex.Unlock() - - _ = ctx - _ = assembleChunks(chunks) - - log.Printf("App '%s' started successfully\n", appName) -} - func assembleChunks(chunks [][]byte) []byte { var wasmBinary []byte for _, chunk := range chunks { @@ -294,51 +270,3 @@ func (c *ChunkPayload) Validate() error { return nil } - -func (p *PropletService) UpdateRegistry(ctx context.Context, registryURL, registryToken string) error { - if registryURL == "" { - return errors.New("registry URL cannot be empty") - } - if _, err := url.ParseRequestURI(registryURL); err != nil { - return fmt.Errorf("invalid registry URL '%s': %w", registryURL, err) - } - - p.registryURL = registryURL - p.registryToken = registryToken - - log.Printf("App Registry updated and persisted: %s\n", registryURL) - - return nil -} - -func (p *PropletService) registryUpdate(ctx context.Context) func(topic string, msg map[string]interface{}) error { - return func(topic string, msg map[string]interface{}) error { - data, err := json.Marshal(msg) - if err != nil { - return err - } - - var payload struct { - RegistryURL string `json:"registry_url"` - RegistryToken string `json:"registry_token"` - } - if err := json.Unmarshal(data, &payload); err != nil { - return err - } - - ackTopic := fmt.Sprintf(RegistryAckTopicTemplate, p.channelID) - - if err := p.UpdateRegistry(ctx, payload.RegistryURL, payload.RegistryToken); err != nil { - if err := p.pubsub.Publish(ctx, ackTopic, map[string]interface{}{"status": "failure", "error": err.Error()}); err != nil { - p.logger.Error("Failed to publish ack message", slog.String("ack_topic", ackTopic), slog.Any("error", err)) - } - } else { - if err := p.pubsub.Publish(ctx, ackTopic, map[string]interface{}{"status": "success"}); err != nil { - p.logger.Error("Failed to publish ack message", slog.String("ack_topic", ackTopic), slog.Any("error", err)) - } - p.logger.Info("App Registry configuration updated successfully", slog.String("ack_topic", ackTopic), slog.String("registry_url", payload.RegistryURL)) - } - - return nil - } -} diff --git a/proxy/config/http.go b/proxy/config/http.go index 8b9901c..6b7336c 100644 --- a/proxy/config/http.go +++ b/proxy/config/http.go @@ -21,12 +21,12 @@ const ( ) type HTTPProxyConfig struct { - RegistryURL string `env:"PROXY_REGISTRY_URL,notEmpty" envDefault:""` - ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` - Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` - Token string `env:"PROXY_REGISTRY_TOKEN" envDefault:""` - Username string `env:"PROXY_REGISTRY_USERNAME" envDefault:""` - Password string `env:"PROXY_REGISTRY_PASSWORD" envDefault:""` + ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` + Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` + Token string `env:"PROXY_REGISTRY_TOKEN" envDefault:""` + Username string `env:"PROXY_REGISTRY_USERNAME" envDefault:""` + Password string `env:"PROXY_REGISTRY_PASSWORD" envDefault:""` + RegistryURL string `env:"PROXY_REGISTRY_URL,notEmpty"` } func (c *HTTPProxyConfig) setupAuthentication(repo *remote.Repository) { @@ -97,7 +97,7 @@ func findLargestLayer(manifest *ocispec.Manifest) (ocispec.Descriptor, error) { return largestLayer, nil } -func createChunks(data []byte, containerName string, chunkSize int) []proplet.ChunkPayload { +func createChunks(data []byte, containerPath string, chunkSize int) []proplet.ChunkPayload { dataSize := len(data) totalChunks := (dataSize + chunkSize - 1) / chunkSize @@ -113,7 +113,7 @@ func createChunks(data []byte, containerName string, chunkSize int) []proplet.Ch log.Printf("Chunk %d size: %d bytes", i, len(chunkData)) chunks = append(chunks, proplet.ChunkPayload{ - AppName: containerName, + AppName: containerPath, ChunkIdx: i, TotalChunks: totalChunks, Data: chunkData, @@ -123,38 +123,36 @@ func createChunks(data []byte, containerName string, chunkSize int) []proplet.Ch return chunks } -func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerName string, chunkSize int) ([]proplet.ChunkPayload, error) { - fullPath := fmt.Sprintf("%s/%s", c.RegistryURL, containerName) - - repo, err := remote.NewRepository(fullPath) +func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerPath string, chunkSize int) ([]proplet.ChunkPayload, error) { + repo, err := remote.NewRepository(containerPath) if err != nil { - return nil, fmt.Errorf("failed to create repository for %s: %w", containerName, err) + return nil, fmt.Errorf("failed to create repository for %s: %w", containerPath, err) } c.setupAuthentication(repo) - manifest, err := c.fetchManifest(ctx, repo, containerName) + manifest, err := c.fetchManifest(ctx, repo, containerPath) if err != nil { return nil, err } largestLayer, err := findLargestLayer(manifest) if err != nil { - return nil, fmt.Errorf("failed to find layer for %s: %w", containerName, err) + return nil, fmt.Errorf("failed to find layer for %s: %w", containerPath, err) } log.Printf("Container size: %d bytes (%.2f MB)", largestLayer.Size, float64(largestLayer.Size)/size) layerReader, err := repo.Fetch(ctx, largestLayer) if err != nil { - return nil, fmt.Errorf("failed to fetch layer for %s: %w", containerName, err) + return nil, fmt.Errorf("failed to fetch layer for %s: %w", containerPath, err) } defer layerReader.Close() data, err := io.ReadAll(layerReader) if err != nil { - return nil, fmt.Errorf("failed to read layer for %s: %w", containerName, err) + return nil, fmt.Errorf("failed to read layer for %s: %w", containerPath, err) } - return createChunks(data, containerName, chunkSize), nil + return createChunks(data, containerPath, chunkSize), nil } diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go index 30031aa..a1f8be2 100644 --- a/proxy/config/mqtt.go +++ b/proxy/config/mqtt.go @@ -1,8 +1,8 @@ package config type MQTTProxyConfig struct { - BrokerURL string `env:"PROXY_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` - Password string `env:"PROXY_PROPLET_KEY,notEmpty" envDefault:""` - PropletID string `env:"PROXY_PROPLET_ID,notEmpty" envDefault:""` - ChannelID string `env:"PROXY_CHANNEL_ID,notEmpty" envDefault:""` + BrokerURL string `env:"PROXY_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` + Password string `env:"PROXY_PROPLET_KEY,notEmpty"` + PropletID string `env:"PROXY_PROPLET_ID,notEmpty" ` + ChannelID string `env:"PROXY_CHANNEL_ID,notEmpty"` } diff --git a/task/task.go b/task/task.go index 44b7a4e..c018fef 100644 --- a/task/task.go +++ b/task/task.go @@ -32,16 +32,17 @@ func (s State) String() string { } type Task struct { - ID string `json:"id"` - Name string `json:"name"` - State State `json:"state"` - File []byte `json:"file,omitempty"` - Inputs []uint64 `json:"inputs,omitempty"` - Results []uint64 `json:"results,omitempty"` - StartTime time.Time `json:"start_time"` - FinishTime time.Time `json:"finish_time"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + ID string `json:"id"` + Name string `json:"name"` + State State `json:"state"` + DownloadFile string `json:"download_file,omitempty"` + File []byte `json:"file,omitempty"` + Inputs []uint64 `json:"inputs,omitempty"` + Results []uint64 `json:"results,omitempty"` + StartTime time.Time `json:"start_time"` + FinishTime time.Time `json:"finish_time"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } type TaskPage struct { diff --git a/test.md b/test.md index 7be33e6..d9023b4 100644 --- a/test.md +++ b/test.md @@ -86,7 +86,7 @@ To start the manager, run the following command ```bash export MANAGER_THING_ID="" export MANAGER_THING_KEY="" -export PRMANAGER_CHANNEL_ID="" +export MANAGER_CHANNEL_ID="" export PROPLET_THING_ID="" export PROPLET_THING_KEY="" propeller-manager @@ -103,22 +103,27 @@ export PROPLET_THING_KEY="" propeller-proplet ``` - **Testing proxy.** - -Start proxy +To start the proxy, run the following command ```bash -go run cmd/proxy/main.go +export PROXY_REGISTRY_URL="" +export PROXY_AUTHENTICATE="TRUE" +export PROXY_REGISTRY_USERNAME="" +export PROXY_REGISTRY_PASSWORD="" +export PROXY_PROPLET_KEY="" +export PROXY_PROPLET_ID="" +export PROXY_CHANNEL_ID="" +propeller-proxy ``` Subscibe to MQTT channel to download the requested binary ```bash -mosquitto_sub -i magistrala -u $PROPLET_THING_ID -P $PROPLET_THING_KEY -t channels/$MANAGER_CHANNEL_ID/messages/registry/server -h localhost +mosquitto_sub -I propeller -u $PROXY_PROPLET_ID -P $PROXY_PROPLET_KEY -t channels/$PROXY_CHANNEL_ID/messages/registry/server -h localhost ``` Publish to MQTT channel to request the container to download ```bash -mosquitto_pub -i magistrala -u $PROPLET_THING_ID -P $PROPLET_THING_KEY -t channels/$MANAGER_CHANNEL_ID/messages/registry/proplet -h localhost -m '{"app_name":"magistrala/users"}' +mosquitto_pub -I propeller -u $PROXY_PROPLET_ID -P $PROXY_PROPLET_KEY -t channels/$PROXY_CHANNEL_ID/messages/registry/proplet -h localhost -m '{"app_name":"mrstevenyaga/add.wasm"}' ``` From b439ca0238b6eecab0faafe83cd512d19f8841a4 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Thu, 19 Dec 2024 19:18:41 +0300 Subject: [PATCH 35/42] remove unused variable Signed-off-by: nyagamunene --- cmd/proplet/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/proplet/main.go b/cmd/proplet/main.go index 7bb28f2..2f1a09e 100644 --- a/cmd/proplet/main.go +++ b/cmd/proplet/main.go @@ -25,7 +25,6 @@ type config struct { MQTTTimeout time.Duration `env:"PROPLET_MQTT_TIMEOUT" envDefault:"30s"` MQTTQoS byte `env:"PROPLET_MQTT_QOS" envDefault:"2"` LivelinessInterval time.Duration `env:"PROPLET_LIVELINESS_INTERVAL" envDefault:"10s"` - RegistryTimeout time.Duration `env:"PROPLET_REGISTRY_TIMEOUT" envDefault:"30s"` ChannelID string `env:"PROPLET_CHANNEL_ID,notEmpty"` ThingID string `env:"PROPLET_THING_ID,notEmpty"` ThingKey string `env:"PROPLET_THING_KEY,notEmpty"` From 6c13917d1b32b348d022fb39fc74426562b92dfc Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 20 Dec 2024 13:42:14 +0300 Subject: [PATCH 36/42] address comments Signed-off-by: nyagamunene --- .golangci.yaml | 1 + cmd/proxy/main.go | 47 ++++++++++++++++++++++++++++---------- proplet/requests.go | 18 +++++++++------ proplet/service.go | 25 ++++++++++---------- proxy/config/mqtt.go | 8 ------- proxy/{config => }/http.go | 32 ++++++++++++++------------ proxy/{mqtt => }/mqtt.go | 23 ++++++++++++------- proxy/service.go | 46 ++++++++++++++++--------------------- task/task.go | 26 ++++++++++----------- task/url.go | 45 ++++++++++++++++++++++++++++++++++++ 10 files changed, 169 insertions(+), 102 deletions(-) delete mode 100644 proxy/config/mqtt.go rename proxy/{config => }/http.go (77%) rename proxy/{mqtt => }/mqtt.go (87%) create mode 100644 task/url.go diff --git a/.golangci.yaml b/.golangci.yaml index 6bf3b58..25c0799 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -34,6 +34,7 @@ linters: - noctx - cyclop - tagalign + - recvcheck linters-settings: gocritic: diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 7c5c90d..40ac1ce 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -8,21 +8,38 @@ import ( "os" "github.com/absmach/propeller/proxy" - "github.com/absmach/propeller/proxy/config" "github.com/caarlos0/env/v11" "golang.org/x/sync/errgroup" ) -const ( - svcName = "proxy" - logLevel = "info" -) +const svcName = "proxy" + +type config struct { + LogLevel string `env:"PROXY_LOG_LEVEL" envDefault:"info"` + + BrokerURL string `env:"PROXY_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` + PropletKey string `env:"PROXY_PROPLET_KEY,notEmpty"` + PropletID string `env:"PROXY_PROPLET_ID,notEmpty" ` + ChannelID string `env:"PROXY_CHANNEL_ID,notEmpty"` + + ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` + Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` + Token string `env:"PROXY_REGISTRY_TOKEN" envDefault:""` + Username string `env:"PROXY_REGISTRY_USERNAME" envDefault:""` + Password string `env:"PROXY_REGISTRY_PASSWORD" envDefault:""` + RegistryURL string `env:"PROXY_REGISTRY_URL,notEmpty"` +} func main() { g, ctx := errgroup.WithContext(context.Background()) + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load configuration : %s", err.Error()) + } + var level slog.Level - if err := level.UnmarshalText([]byte(logLevel)); err != nil { + if err := level.UnmarshalText([]byte(cfg.LogLevel)); err != nil { log.Fatalf("failed to parse log level: %s", err.Error()) } logHandler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ @@ -31,14 +48,20 @@ func main() { logger := slog.New(logHandler) slog.SetDefault(logger) - mqttCfg := config.MQTTProxyConfig{} - if err := env.Parse(&mqttCfg); err != nil { - logger.Error("failed to load mqtt config", slog.Any("error", err)) + mqttCfg := proxy.MQTTProxyConfig{ + BrokerURL: cfg.BrokerURL, + Password: cfg.PropletKey, + PropletID: cfg.PropletID, + ChannelID: cfg.ChannelID, } - httpCfg := config.HTTPProxyConfig{} - if err := env.Parse(&httpCfg); err != nil { - logger.Error("failed to load http config", slog.Any("error", err)) + httpCfg := proxy.HTTPProxyConfig{ + ChunkSize: cfg.ChunkSize, + Authenticate: cfg.Authenticate, + Token: cfg.Token, + Username: cfg.Username, + Password: cfg.Password, + RegistryURL: cfg.RegistryURL, } logger.Info("successfully initialized MQTT and HTTP config") diff --git a/proplet/requests.go b/proplet/requests.go index 8d42405..12d5687 100644 --- a/proplet/requests.go +++ b/proplet/requests.go @@ -1,13 +1,17 @@ package proplet -import "errors" +import ( + "errors" + + "github.com/absmach/propeller/task" +) type startRequest struct { - ID string - FunctionName string - WasmFile []byte - WasmFileDownloadPath string - Params []uint64 + ID string + FunctionName string + WasmFile []byte + imageURL task.URLValue + Params []uint64 } func (r startRequest) Validate() error { @@ -17,7 +21,7 @@ func (r startRequest) Validate() error { if r.FunctionName == "" { return errors.New("function name is required") } - if r.WasmFile == nil && r.WasmFileDownloadPath == "" { + if r.WasmFile == nil && r.imageURL == (task.URLValue{}) { return errors.New("either wasm file or wasm file download path is required") } diff --git a/proplet/service.go b/proplet/service.go index f5428c3..77b0bfe 100644 --- a/proplet/service.go +++ b/proplet/service.go @@ -138,11 +138,11 @@ func (p *PropletService) handleStartCommand(ctx context.Context) func(topic stri } req := startRequest{ - ID: payload.ID, - FunctionName: payload.Name, - WasmFile: payload.File, - WasmFileDownloadPath: payload.DownloadFile, - Params: payload.Inputs, + ID: payload.ID, + FunctionName: payload.Name, + WasmFile: payload.File, + imageURL: payload.ImageURL, + Params: payload.Inputs, } if err := req.Validate(); err != nil { return err @@ -159,7 +159,7 @@ func (p *PropletService) handleStartCommand(ctx context.Context) func(topic stri } pl := map[string]interface{}{ - "app_name": req.WasmFileDownloadPath, + "app_name": req.imageURL, } tp := fmt.Sprintf(fetchRequestTopicTemplate, p.channelID) if err := p.pubsub.Publish(ctx, tp, pl); err != nil { @@ -167,19 +167,20 @@ func (p *PropletService) handleStartCommand(ctx context.Context) func(topic stri } go func() { - p.logger.Info("Waiting for chunks", slog.String("app_name", req.WasmFileDownloadPath)) + p.logger.Info("Waiting for chunks", slog.String("app_name", req.imageURL.String())) for { p.chunksMutex.Lock() - metadata, exists := p.chunkMetadata[req.WasmFileDownloadPath] - receivedChunks := len(p.chunks[req.WasmFileDownloadPath]) + urlStr := req.imageURL.String() + metadata, exists := p.chunkMetadata[urlStr] + receivedChunks := len(p.chunks[urlStr]) p.chunksMutex.Unlock() if exists && receivedChunks == metadata.TotalChunks { - p.logger.Info("All chunks received, deploying app", slog.String("app_name", req.WasmFileDownloadPath)) - wasmBinary := assembleChunks(p.chunks[req.WasmFileDownloadPath]) + p.logger.Info("All chunks received, deploying app", slog.String("app_name", urlStr)) + wasmBinary := assembleChunks(p.chunks[urlStr]) if err := p.runtime.StartApp(ctx, wasmBinary, req.ID, req.FunctionName, req.Params...); err != nil { - p.logger.Error("Failed to start app", slog.String("app_name", req.WasmFileDownloadPath), slog.Any("error", err)) + p.logger.Error("Failed to start app", slog.String("app_name", urlStr), slog.Any("error", err)) } break diff --git a/proxy/config/mqtt.go b/proxy/config/mqtt.go deleted file mode 100644 index a1f8be2..0000000 --- a/proxy/config/mqtt.go +++ /dev/null @@ -1,8 +0,0 @@ -package config - -type MQTTProxyConfig struct { - BrokerURL string `env:"PROXY_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` - Password string `env:"PROXY_PROPLET_KEY,notEmpty"` - PropletID string `env:"PROXY_PROPLET_ID,notEmpty" ` - ChannelID string `env:"PROXY_CHANNEL_ID,notEmpty"` -} diff --git a/proxy/config/http.go b/proxy/http.go similarity index 77% rename from proxy/config/http.go rename to proxy/http.go index 6b7336c..db69c4e 100644 --- a/proxy/config/http.go +++ b/proxy/http.go @@ -1,4 +1,4 @@ -package config +package proxy import ( "context" @@ -9,6 +9,7 @@ import ( "log" "github.com/absmach/propeller/proplet" + "github.com/absmach/propeller/task" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" @@ -21,12 +22,12 @@ const ( ) type HTTPProxyConfig struct { - ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` - Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` - Token string `env:"PROXY_REGISTRY_TOKEN" envDefault:""` - Username string `env:"PROXY_REGISTRY_USERNAME" envDefault:""` - Password string `env:"PROXY_REGISTRY_PASSWORD" envDefault:""` - RegistryURL string `env:"PROXY_REGISTRY_URL,notEmpty"` + ChunkSize int + Authenticate bool + Token string + Username string + Password string + RegistryURL string } func (c *HTTPProxyConfig) setupAuthentication(repo *remote.Repository) { @@ -123,36 +124,37 @@ func createChunks(data []byte, containerPath string, chunkSize int) []proplet.Ch return chunks } -func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerPath string, chunkSize int) ([]proplet.ChunkPayload, error) { - repo, err := remote.NewRepository(containerPath) +func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerPath task.URLValue, chunkSize int) ([]proplet.ChunkPayload, error) { + reference := containerPath.String() + repo, err := remote.NewRepository(reference) if err != nil { - return nil, fmt.Errorf("failed to create repository for %s: %w", containerPath, err) + return nil, fmt.Errorf("failed to create repository for %s: %w", reference, err) } c.setupAuthentication(repo) - manifest, err := c.fetchManifest(ctx, repo, containerPath) + manifest, err := c.fetchManifest(ctx, repo, reference) if err != nil { return nil, err } largestLayer, err := findLargestLayer(manifest) if err != nil { - return nil, fmt.Errorf("failed to find layer for %s: %w", containerPath, err) + return nil, fmt.Errorf("failed to find layer for %s: %w", reference, err) } log.Printf("Container size: %d bytes (%.2f MB)", largestLayer.Size, float64(largestLayer.Size)/size) layerReader, err := repo.Fetch(ctx, largestLayer) if err != nil { - return nil, fmt.Errorf("failed to fetch layer for %s: %w", containerPath, err) + return nil, fmt.Errorf("failed to fetch layer for %s: %w", reference, err) } defer layerReader.Close() data, err := io.ReadAll(layerReader) if err != nil { - return nil, fmt.Errorf("failed to read layer for %s: %w", containerPath, err) + return nil, fmt.Errorf("failed to read layer for %s: %w", reference, err) } - return createChunks(data, containerPath, chunkSize), nil + return createChunks(data, reference, chunkSize), nil } diff --git a/proxy/mqtt/mqtt.go b/proxy/mqtt.go similarity index 87% rename from proxy/mqtt/mqtt.go rename to proxy/mqtt.go index e1afb0e..2bd9279 100644 --- a/proxy/mqtt/mqtt.go +++ b/proxy/mqtt.go @@ -1,4 +1,4 @@ -package mqtt +package proxy import ( "context" @@ -8,10 +8,17 @@ import ( "time" "github.com/absmach/propeller/proplet" - "github.com/absmach/propeller/proxy/config" + "github.com/absmach/propeller/task" mqtt "github.com/eclipse/paho.mqtt.golang" ) +type MQTTProxyConfig struct { + BrokerURL string + Password string + PropletID string + ChannelID string +} + const ( connTimeout = 10 reconnTimeout = 1 @@ -22,10 +29,10 @@ const ( type RegistryClient struct { client mqtt.Client - config *config.MQTTProxyConfig + config *MQTTProxyConfig } -func NewMQTTClient(cfg *config.MQTTProxyConfig) (*RegistryClient, error) { +func NewMQTTClient(cfg *MQTTProxyConfig) (*RegistryClient, error) { opts := mqtt.NewClientOptions(). AddBroker(cfg.BrokerURL). SetClientID("Proplet-" + cfg.PropletID). @@ -71,14 +78,14 @@ func (c *RegistryClient) Connect(ctx context.Context) error { return nil } -func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- string) error { +func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- task.URLValue) error { handler := func(client mqtt.Client, msg mqtt.Message) { data := msg.Payload() payLoad := struct { - Appname string `json:"app_name"` + Appname task.URLValue `json:"app_name"` }{ - Appname: "", + Appname: task.URLValue{}, } err := json.Unmarshal(data, &payLoad) @@ -90,7 +97,7 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- str select { case containerChan <- payLoad.Appname: - log.Printf("Received container request: %s", payLoad.Appname) + log.Printf("Received container request: %s", payLoad.Appname.String()) case <-ctx.Done(): return diff --git a/proxy/service.go b/proxy/service.go index 74d5e47..439e79a 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -6,42 +6,39 @@ import ( "log/slog" "github.com/absmach/propeller/proplet" - "github.com/absmach/propeller/proxy/config" - "github.com/absmach/propeller/proxy/mqtt" + "github.com/absmach/propeller/task" ) const chunkBuffer = 10 type ProxyService struct { - orasconfig *config.HTTPProxyConfig - mqttClient *mqtt.RegistryClient + orasconfig *HTTPProxyConfig + mqttClient *RegistryClient logger *slog.Logger - containerChan chan string + containerChan chan task.URLValue dataChan chan proplet.ChunkPayload } -func NewService(ctx context.Context, mqttCfg *config.MQTTProxyConfig, httpCfg *config.HTTPProxyConfig, logger *slog.Logger) (*ProxyService, error) { - mqttClient, err := mqtt.NewMQTTClient(mqttCfg) +func NewService(ctx context.Context, mqttCfg *MQTTProxyConfig, httpCfg *HTTPProxyConfig, logger *slog.Logger) (*ProxyService, error) { + mqttClient, err := NewMQTTClient(mqttCfg) if err != nil { return nil, fmt.Errorf("failed to initialize MQTT client: %w", err) } - logger.Info("successfully initialized MQTT client") - return &ProxyService{ orasconfig: httpCfg, mqttClient: mqttClient, logger: logger, - containerChan: make(chan string, 1), + containerChan: make(chan task.URLValue, 1), dataChan: make(chan proplet.ChunkPayload, chunkBuffer), }, nil } -func (s *ProxyService) MQTTClient() *mqtt.RegistryClient { +func (s *ProxyService) MQTTClient() *RegistryClient { return s.mqttClient } -func (s *ProxyService) ContainerChan() chan string { +func (s *ProxyService) ContainerChan() chan task.URLValue { return s.containerChan } @@ -53,7 +50,9 @@ func (s *ProxyService) StreamHTTP(ctx context.Context) error { case containerName := <-s.containerChan: chunks, err := s.orasconfig.FetchFromReg(ctx, containerName, s.orasconfig.ChunkSize) if err != nil { - s.logger.Error("failed to fetch container", "container", containerName, "error", err) + s.logger.Error("failed to fetch container", + slog.Any("container name", containerName), + slog.Any("error", err)) continue } @@ -63,9 +62,9 @@ func (s *ProxyService) StreamHTTP(ctx context.Context) error { select { case s.dataChan <- chunk: s.logger.Info("sent container chunk to MQTT stream", - "container", containerName, - "chunk", chunk.ChunkIdx, - "total", chunk.TotalChunks) + slog.Any("container", containerName), + slog.Int("chunk", chunk.ChunkIdx), + slog.Int("total", chunk.TotalChunks)) case <-ctx.Done(): return ctx.Err() } @@ -84,24 +83,19 @@ func (s *ProxyService) StreamMQTT(ctx context.Context) error { case chunk := <-s.dataChan: if err := s.mqttClient.PublishContainer(ctx, chunk); err != nil { s.logger.Error("failed to publish container chunk", - "error", err, - "chunk", chunk.ChunkIdx, - "total", chunk.TotalChunks) + slog.Any("error", err), + slog.Int("chunk", chunk.ChunkIdx), + slog.Int("total", chunk.TotalChunks)) continue } - s.logger.Info("published container chunk", - "chunk_name", chunk.AppName, - "chunk_no", chunk.ChunkIdx, - "total", chunk.TotalChunks) - containerChunks[chunk.AppName]++ if containerChunks[chunk.AppName] == chunk.TotalChunks { s.logger.Info("successfully sent all chunks", - "container", chunk.AppName, - "total_chunks", chunk.TotalChunks) + slog.String("container", chunk.AppName), + slog.Int("total_chunks", chunk.TotalChunks)) delete(containerChunks, chunk.AppName) } } diff --git a/task/task.go b/task/task.go index c018fef..daf0595 100644 --- a/task/task.go +++ b/task/task.go @@ -1,8 +1,6 @@ package task -import ( - "time" -) +import "time" type State uint8 @@ -32,17 +30,17 @@ func (s State) String() string { } type Task struct { - ID string `json:"id"` - Name string `json:"name"` - State State `json:"state"` - DownloadFile string `json:"download_file,omitempty"` - File []byte `json:"file,omitempty"` - Inputs []uint64 `json:"inputs,omitempty"` - Results []uint64 `json:"results,omitempty"` - StartTime time.Time `json:"start_time"` - FinishTime time.Time `json:"finish_time"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + ID string `json:"id"` + Name string `json:"name"` + State State `json:"state"` + ImageURL URLValue `json:"image_url,omitempty"` + File []byte `json:"file,omitempty"` + Inputs []uint64 `json:"inputs,omitempty"` + Results []uint64 `json:"results,omitempty"` + StartTime time.Time `json:"start_time"` + FinishTime time.Time `json:"finish_time"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } type TaskPage struct { diff --git a/task/url.go b/task/url.go new file mode 100644 index 0000000..2f70d13 --- /dev/null +++ b/task/url.go @@ -0,0 +1,45 @@ +package task + +import ( + "encoding/json" + "net/url" +) + +type URLValue url.URL + +func (u URLValue) MarshalJSON() ([]byte, error) { + return json.Marshal((*url.URL)(&u).String()) +} + +func (u *URLValue) UnmarshalJSON(data []byte) error { + var urlStr string + if err := json.Unmarshal(data, &urlStr); err != nil { + return err + } + + if urlStr == "" { + return nil + } + + parsedURL, err := url.Parse(urlStr) + if err != nil { + return err + } + + *u = URLValue(*parsedURL) + + return nil +} + +func (u *URLValue) URL() *url.URL { + if u == nil { + return nil + } + val := url.URL(*u) + + return &val +} + +func (u URLValue) String() string { + return (*url.URL)(&u).String() +} From 03b75aaa9abf4eaa7fc69c5f61769c47b85a9361 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 20 Dec 2024 13:44:24 +0300 Subject: [PATCH 37/42] remove linter check Signed-off-by: nyagamunene --- .golangci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index 25c0799..6bf3b58 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -34,7 +34,6 @@ linters: - noctx - cyclop - tagalign - - recvcheck linters-settings: gocritic: From a1f49401ba8a2daeff9e9efb652a7c0ab51b55fa Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 20 Dec 2024 14:55:36 +0300 Subject: [PATCH 38/42] address comments Signed-off-by: nyagamunene --- cmd/proxy/main.go | 8 ++++---- proplet/requests.go | 6 ++---- proplet/service.go | 13 ++++++------- proxy/http.go | 19 ++++++++----------- proxy/mqtt.go | 9 ++++----- proxy/service.go | 7 +++---- task/task.go | 2 +- task/url.go | 45 --------------------------------------------- test.md | 25 ------------------------- 9 files changed, 28 insertions(+), 106 deletions(-) delete mode 100644 task/url.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 40ac1ce..e70f93a 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -17,10 +17,10 @@ const svcName = "proxy" type config struct { LogLevel string `env:"PROXY_LOG_LEVEL" envDefault:"info"` - BrokerURL string `env:"PROXY_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` - PropletKey string `env:"PROXY_PROPLET_KEY,notEmpty"` - PropletID string `env:"PROXY_PROPLET_ID,notEmpty" ` - ChannelID string `env:"PROXY_CHANNEL_ID,notEmpty"` + BrokerURL string `env:"PROPLET_MQTT_ADDRESS" envDefault:"tcp://localhost:1883"` + PropletKey string `env:"PROPLET_THING_KEY,notEmpty"` + PropletID string `env:"PROPLET_THING_ID,notEmpty" ` + ChannelID string `env:"PROPLET_CHANNEL_ID,notEmpty"` ChunkSize int `env:"PROXY_CHUNK_SIZE" envDefault:"512000"` Authenticate bool `env:"PROXY_AUTHENTICATE" envDefault:"false"` diff --git a/proplet/requests.go b/proplet/requests.go index 12d5687..48a05b4 100644 --- a/proplet/requests.go +++ b/proplet/requests.go @@ -2,15 +2,13 @@ package proplet import ( "errors" - - "github.com/absmach/propeller/task" ) type startRequest struct { ID string FunctionName string WasmFile []byte - imageURL task.URLValue + imageURL string Params []uint64 } @@ -21,7 +19,7 @@ func (r startRequest) Validate() error { if r.FunctionName == "" { return errors.New("function name is required") } - if r.WasmFile == nil && r.imageURL == (task.URLValue{}) { + if r.WasmFile == nil && r.imageURL == "" { return errors.New("either wasm file or wasm file download path is required") } diff --git a/proplet/service.go b/proplet/service.go index 77b0bfe..13b3d7b 100644 --- a/proplet/service.go +++ b/proplet/service.go @@ -167,20 +167,19 @@ func (p *PropletService) handleStartCommand(ctx context.Context) func(topic stri } go func() { - p.logger.Info("Waiting for chunks", slog.String("app_name", req.imageURL.String())) + p.logger.Info("Waiting for chunks", slog.String("app_name", req.imageURL)) for { p.chunksMutex.Lock() - urlStr := req.imageURL.String() - metadata, exists := p.chunkMetadata[urlStr] - receivedChunks := len(p.chunks[urlStr]) + metadata, exists := p.chunkMetadata[req.imageURL] + receivedChunks := len(p.chunks[req.imageURL]) p.chunksMutex.Unlock() if exists && receivedChunks == metadata.TotalChunks { - p.logger.Info("All chunks received, deploying app", slog.String("app_name", urlStr)) - wasmBinary := assembleChunks(p.chunks[urlStr]) + p.logger.Info("All chunks received, deploying app", slog.String("app_name", req.imageURL)) + wasmBinary := assembleChunks(p.chunks[req.imageURL]) if err := p.runtime.StartApp(ctx, wasmBinary, req.ID, req.FunctionName, req.Params...); err != nil { - p.logger.Error("Failed to start app", slog.String("app_name", urlStr), slog.Any("error", err)) + p.logger.Error("Failed to start app", slog.String("app_name", req.imageURL), slog.Any("error", err)) } break diff --git a/proxy/http.go b/proxy/http.go index db69c4e..6c288c9 100644 --- a/proxy/http.go +++ b/proxy/http.go @@ -9,7 +9,6 @@ import ( "log" "github.com/absmach/propeller/proplet" - "github.com/absmach/propeller/task" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" @@ -43,7 +42,6 @@ func (c *HTTPProxyConfig) setupAuthentication(repo *remote.Repository) { } } else if c.Token != "" { cred = auth.Credential{ - Username: c.Username, AccessToken: c.Token, } } @@ -124,37 +122,36 @@ func createChunks(data []byte, containerPath string, chunkSize int) []proplet.Ch return chunks } -func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerPath task.URLValue, chunkSize int) ([]proplet.ChunkPayload, error) { - reference := containerPath.String() - repo, err := remote.NewRepository(reference) +func (c *HTTPProxyConfig) FetchFromReg(ctx context.Context, containerPath string, chunkSize int) ([]proplet.ChunkPayload, error) { + repo, err := remote.NewRepository(containerPath) if err != nil { - return nil, fmt.Errorf("failed to create repository for %s: %w", reference, err) + return nil, fmt.Errorf("failed to create repository for %s: %w", containerPath, err) } c.setupAuthentication(repo) - manifest, err := c.fetchManifest(ctx, repo, reference) + manifest, err := c.fetchManifest(ctx, repo, containerPath) if err != nil { return nil, err } largestLayer, err := findLargestLayer(manifest) if err != nil { - return nil, fmt.Errorf("failed to find layer for %s: %w", reference, err) + return nil, fmt.Errorf("failed to find layer for %s: %w", containerPath, err) } log.Printf("Container size: %d bytes (%.2f MB)", largestLayer.Size, float64(largestLayer.Size)/size) layerReader, err := repo.Fetch(ctx, largestLayer) if err != nil { - return nil, fmt.Errorf("failed to fetch layer for %s: %w", reference, err) + return nil, fmt.Errorf("failed to fetch layer for %s: %w", containerPath, err) } defer layerReader.Close() data, err := io.ReadAll(layerReader) if err != nil { - return nil, fmt.Errorf("failed to read layer for %s: %w", reference, err) + return nil, fmt.Errorf("failed to read layer for %s: %w", containerPath, err) } - return createChunks(data, reference, chunkSize), nil + return createChunks(data, containerPath, chunkSize), nil } diff --git a/proxy/mqtt.go b/proxy/mqtt.go index 2bd9279..82c69ae 100644 --- a/proxy/mqtt.go +++ b/proxy/mqtt.go @@ -8,7 +8,6 @@ import ( "time" "github.com/absmach/propeller/proplet" - "github.com/absmach/propeller/task" mqtt "github.com/eclipse/paho.mqtt.golang" ) @@ -78,14 +77,14 @@ func (c *RegistryClient) Connect(ctx context.Context) error { return nil } -func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- task.URLValue) error { +func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- string) error { handler := func(client mqtt.Client, msg mqtt.Message) { data := msg.Payload() payLoad := struct { - Appname task.URLValue `json:"app_name"` + Appname string `json:"app_name"` }{ - Appname: task.URLValue{}, + Appname: "", } err := json.Unmarshal(data, &payLoad) @@ -97,7 +96,7 @@ func (c *RegistryClient) Subscribe(ctx context.Context, containerChan chan<- tas select { case containerChan <- payLoad.Appname: - log.Printf("Received container request: %s", payLoad.Appname.String()) + log.Printf("Received container request: %s", payLoad.Appname) case <-ctx.Done(): return diff --git a/proxy/service.go b/proxy/service.go index 439e79a..c8fe28c 100644 --- a/proxy/service.go +++ b/proxy/service.go @@ -6,7 +6,6 @@ import ( "log/slog" "github.com/absmach/propeller/proplet" - "github.com/absmach/propeller/task" ) const chunkBuffer = 10 @@ -15,7 +14,7 @@ type ProxyService struct { orasconfig *HTTPProxyConfig mqttClient *RegistryClient logger *slog.Logger - containerChan chan task.URLValue + containerChan chan string dataChan chan proplet.ChunkPayload } @@ -29,7 +28,7 @@ func NewService(ctx context.Context, mqttCfg *MQTTProxyConfig, httpCfg *HTTPProx orasconfig: httpCfg, mqttClient: mqttClient, logger: logger, - containerChan: make(chan task.URLValue, 1), + containerChan: make(chan string, 1), dataChan: make(chan proplet.ChunkPayload, chunkBuffer), }, nil } @@ -38,7 +37,7 @@ func (s *ProxyService) MQTTClient() *RegistryClient { return s.mqttClient } -func (s *ProxyService) ContainerChan() chan task.URLValue { +func (s *ProxyService) ContainerChan() chan string { return s.containerChan } diff --git a/task/task.go b/task/task.go index daf0595..fbac322 100644 --- a/task/task.go +++ b/task/task.go @@ -33,7 +33,7 @@ type Task struct { ID string `json:"id"` Name string `json:"name"` State State `json:"state"` - ImageURL URLValue `json:"image_url,omitempty"` + ImageURL string `json:"image_url,omitempty"` File []byte `json:"file,omitempty"` Inputs []uint64 `json:"inputs,omitempty"` Results []uint64 `json:"results,omitempty"` diff --git a/task/url.go b/task/url.go deleted file mode 100644 index 2f70d13..0000000 --- a/task/url.go +++ /dev/null @@ -1,45 +0,0 @@ -package task - -import ( - "encoding/json" - "net/url" -) - -type URLValue url.URL - -func (u URLValue) MarshalJSON() ([]byte, error) { - return json.Marshal((*url.URL)(&u).String()) -} - -func (u *URLValue) UnmarshalJSON(data []byte) error { - var urlStr string - if err := json.Unmarshal(data, &urlStr); err != nil { - return err - } - - if urlStr == "" { - return nil - } - - parsedURL, err := url.Parse(urlStr) - if err != nil { - return err - } - - *u = URLValue(*parsedURL) - - return nil -} - -func (u *URLValue) URL() *url.URL { - if u == nil { - return nil - } - val := url.URL(*u) - - return &val -} - -func (u URLValue) String() string { - return (*url.URL)(&u).String() -} diff --git a/test.md b/test.md index d9023b4..306d94e 100644 --- a/test.md +++ b/test.md @@ -102,28 +102,3 @@ export PROPLET_THING_ID="" export PROPLET_THING_KEY="" propeller-proplet ``` - -To start the proxy, run the following command - -```bash -export PROXY_REGISTRY_URL="" -export PROXY_AUTHENTICATE="TRUE" -export PROXY_REGISTRY_USERNAME="" -export PROXY_REGISTRY_PASSWORD="" -export PROXY_PROPLET_KEY="" -export PROXY_PROPLET_ID="" -export PROXY_CHANNEL_ID="" -propeller-proxy -``` - -Subscibe to MQTT channel to download the requested binary - -```bash -mosquitto_sub -I propeller -u $PROXY_PROPLET_ID -P $PROXY_PROPLET_KEY -t channels/$PROXY_CHANNEL_ID/messages/registry/server -h localhost -``` - -Publish to MQTT channel to request the container to download - -```bash -mosquitto_pub -I propeller -u $PROXY_PROPLET_ID -P $PROXY_PROPLET_KEY -t channels/$PROXY_CHANNEL_ID/messages/registry/proplet -h localhost -m '{"app_name":"mrstevenyaga/add.wasm"}' -``` From cf7a9ebfedfea846a96cc693b101fa81d27f5c0c Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 20 Dec 2024 14:57:49 +0300 Subject: [PATCH 39/42] remove test.md file Signed-off-by: nyagamunene --- test.md | 104 -------------------------------------------------------- 1 file changed, 104 deletions(-) delete mode 100644 test.md diff --git a/test.md b/test.md deleted file mode 100644 index 306d94e..0000000 --- a/test.md +++ /dev/null @@ -1,104 +0,0 @@ -# Test - -Start docker composition - -```bash -cd docker -docker compose up -``` - -Login as admin user - -```bash -USER_TOKEN=$(magistrala-cli users token admin 12345678 | jq -r .access_token) -``` - -Create a domain - -```bash -DOMAIN_ID=$(magistrala-cli domains create demo demo $USER_TOKEN | jq -r .id) -``` - -Create a thing called manager - -```bash -magistrala-cli things create '{"name": "Propeller Manager", "tags": ["manager", "propeller"], "status": "enabled"}' $DOMAIN_ID $USER_TOKEN -``` - -Set the following environment variables from the respose - -```bash -export MANAGER_THING_ID="" -export MANAGER_THING_KEY="" -``` - -Create a channel called manager - -```bash -magistrala-cli channels create '{"name": "Propeller Manager", "tags": ["manager", "propeller"], "status": "enabled"}' $DOMAIN_ID $USER_TOKEN -``` - -Set the following environment variables from the respose - -```bash -export MANAGER_CHANNEL_ID="" -``` - -Connect the thing to the manager channel - -```bash -magistrala-cli things connect $MANAGER_THING_ID $MANAGER_CHANNEL_ID $DOMAIN_ID $USER_TOKEN -``` - -Create a thing called proplet - -```bash -magistrala-cli things create '{"name": "Propeller Proplet", "tags": ["proplet", "propeller"], "status": "enabled"}' $DOMAIN_ID $USER_TOKEN -``` - -Set the following environment variables from the respose - -```bash -export PROPLET_THING_ID="" -export PROPLET_THING_KEY="" -``` - -Connect the thing to the manager channel - -```bash -magistrala-cli things connect $PROPLET_THING_ID $MANAGER_CHANNEL_ID $DOMAIN_ID $USER_TOKEN -``` - -Publish create message to the manager channel. This creates a new proplet. - -```bash -mosquitto_pub -u $PROPLET_THING_ID -P $PROPLET_THING_KEY -I propeller -t channels/$MANAGER_CHANNEL_ID/messages/control/proplet/create -h localhost -m "{\"proplet_id\": \"$PROPLET_THING_ID\", \"name\": \"proplet-1\"}" -``` - -Publish alive message to the manager channel. This updates the proplet. - -```bash -mosquitto_pub -u $PROPLET_THING_ID -P $PROPLET_THING_KEY -I propeller -t channels/$MANAGER_CHANNEL_ID/messages/control/proplet/alive -h localhost -m "{\"proplet_id\": \"$PROPLET_THING_ID\"}" -``` - -To start the manager, run the following command - -```bash -export MANAGER_THING_ID="" -export MANAGER_THING_KEY="" -export MANAGER_CHANNEL_ID="" -export PROPLET_THING_ID="" -export PROPLET_THING_KEY="" -propeller-manager -``` - -To start the proplet, run the following command - -```bash -export MANAGER_THING_ID="" -export MANAGER_THING_KEY="" -export PROPLET_CHANNEL_ID="" -export PROPLET_THING_ID="" -export PROPLET_THING_KEY="" -propeller-proplet -``` From 0f703983f26c402e2c5781de0eece89d160f9f35 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 20 Dec 2024 15:46:42 +0300 Subject: [PATCH 40/42] update make install command Signed-off-by: nyagamunene --- Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 30d9855..5d0a309 100644 --- a/Makefile +++ b/Makefile @@ -21,10 +21,8 @@ $(SERVICES): $(call compile_service,$(@)) install: - for file in $(BUILD_DIR)/*; do \ - if [ "$$file" = "$${file%%.wasm}" ]; then \ - cp "$$file" "$(GOBIN)/propeller-$$(basename "$$file")"; \ - fi; \ + for file in $(wildcard $(BUILD_DIR)/*[!.wasm]); do \ + cp "$$file" "$(GOBIN)/propeller-$$(basename "$$file")"; \ done .PHONY: all $(SERVICES) From febf61c4a0fe75408f37281d521dbd9c557c5b55 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 20 Dec 2024 18:33:07 +0300 Subject: [PATCH 41/42] update make install command Signed-off-by: nyagamunene --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5d0a309..10085e8 100644 --- a/Makefile +++ b/Makefile @@ -21,9 +21,7 @@ $(SERVICES): $(call compile_service,$(@)) install: - for file in $(wildcard $(BUILD_DIR)/*[!.wasm]); do \ - cp "$$file" "$(GOBIN)/propeller-$$(basename "$$file")"; \ - done + $(foreach f,$(wildcard $(BUILD_DIR)/*[!.wasm]),cp $(f) $(patsubst $(BUILD_DIR)/%,$(GOBIN)/propeller-%,$(f));) .PHONY: all $(SERVICES) all: $(SERVICES) From 2da6d9706654d90d4a1f58762f48af60bde5e305 Mon Sep 17 00:00:00 2001 From: nyagamunene Date: Fri, 20 Dec 2024 18:43:03 +0300 Subject: [PATCH 42/42] add comments to make install command Signed-off-by: nyagamunene --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 10085e8..9b222b2 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ endef $(SERVICES): $(call compile_service,$(@)) + +# Install all non-WASM executables from the build directory to GOBIN with 'propeller-' prefix install: $(foreach f,$(wildcard $(BUILD_DIR)/*[!.wasm]),cp $(f) $(patsubst $(BUILD_DIR)/%,$(GOBIN)/propeller-%,$(f));)