From fb4e3ef131f2cfe56ca415212231533be1d7f36d Mon Sep 17 00:00:00 2001 From: Olivier Barais Date: Mon, 25 Nov 2024 16:29:48 +0100 Subject: [PATCH] introduce a way to remain below a threshold for the mail throughput --- Dockerfile | 25 +++++++++++++++++++ README.md | 3 ++- go.mod | 10 +++++--- go.sum | 20 +++++++++++++++ main.go | 22 ++++++++--------- ratelimiter.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ remotes.go | 38 ++++++++++++++++++++++------ send4mail.sh | 6 +++++ smtp.go | 25 +++++++++++++++++++ smtprelay.ini | 14 +++++++---- 10 files changed, 202 insertions(+), 28 deletions(-) create mode 100644 Dockerfile create mode 100644 ratelimiter.go create mode 100644 send4mail.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6b9dcc2f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM golang:1.22-alpine AS build +WORKDIR /app + + + +COPY go.mod ./ +COPY go.sum ./ + +RUN apk add git + +RUN go mod download + +COPY *.go ./ + +RUN go build -o smtprelay + +FROM golang:1.22-alpine + +WORKDIR /app + +COPY --from=build /app/smtprelay ./ + +EXPOSE 25 + +CMD ["./smtprelay", "-config", "smtprelay.ini"] diff --git a/README.md b/README.md index db81740f..2f8c0fa9 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/decke/smtprelay)](https://goreportcard.com/report/github.com/decke/smtprelay) Simple Golang based SMTP relay/proxy server that accepts mail via SMTP -and forwards it directly to another SMTP server. +and forwards it directly to another SMTP server. Fork to add the ability to cache mail that can not be sent due to rate limit. Mail are sent when the the service will not exceed the rate limit. ## Why another SMTP server? @@ -30,3 +30,4 @@ device which produces mail. * Forwards all mail to a smarthost (any SMTP server) * Small codebase * IPv6 support +* Cache mail to avoid exceeding the rate limit per remote diff --git a/go.mod b/go.mod index 6e21647d..c1c4d61e 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,23 @@ module github.com/decke/smtprelay require ( github.com/chrj/smtpd v0.3.1 github.com/google/uuid v1.6.0 + github.com/maypok86/otter v1.2.4 github.com/peterbourgon/ff/v3 v3.4.0 + github.com/sethvargo/go-limiter v1.0.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 - golang.org/x/crypto v0.28.0 + golang.org/x/crypto v0.29.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dolthub/maphash v0.1.0 // indirect + github.com/gammazero/deque v1.0.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.27.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) -go 1.20 +go 1.22 diff --git a/go.sum b/go.sum index 47c59898..de623e5e 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ= +github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4= +github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= +github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -13,6 +17,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/maypok86/otter v1.2.3 h1:jxyPD4ofCwtrQM5is5JNrdAs+6+JQkf/PREZd7JCVgg= +github.com/maypok86/otter v1.2.3/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4= +github.com/maypok86/otter v1.2.4-0.20241122154217-c7fa1631301b h1:OcjzyR4TevoH7W/4WIH4ymBR0RCVoRJrvRFU1bW/SmI= +github.com/maypok86/otter v1.2.4-0.20241122154217-c7fa1631301b/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4= +github.com/maypok86/otter v1.2.4 h1:HhW1Pq6VdJkmWwcZZq19BlEQkHtI8xgsQzBVXJU0nfc= +github.com/maypok86/otter v1.2.4/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4= github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -20,17 +30,27 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/scalalang2/golang-fifo v1.0.2 h1:sfOJBB86iXuqB5WoLtVI7+wxn8UOEOr9SnJaTakinBA= +github.com/scalalang2/golang-fifo v1.0.2/go.mod h1:TsyVkLbka5m8tmfqsWBXwJ7Om1jV/uuOuvoPulZbMmA= +github.com/sethvargo/go-limiter v1.0.0 h1:JqW13eWEMn0VFv86OKn8wiYJY/m250WoXdrjRV0kLe4= +github.com/sethvargo/go-limiter v1.0.0/go.mod h1:01b6tW25Ap+MeLYBuD4aHunMrJoNO5PVUFdS9rac3II= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/main.go b/main.go index 65524dbb..aec6e901 100644 --- a/main.go +++ b/main.go @@ -195,7 +195,7 @@ func mailHandler(peer smtpd.Peer, env smtpd.Envelope) error { environ = append(environ, fmt.Sprintf("%s=%s", "SMTPRELAY_PEER", peerIP)) cmd := exec.Cmd{ - Env: environ, + Env: environ, Path: *command, } @@ -211,7 +211,7 @@ func mailHandler(peer smtpd.Peer, env smtpd.Envelope) error { cmdLogger.Info("pipe command successful: " + stdout.String()) } - + var smtpError *smtpd.Error for _, remote := range envRemotes { logger = logger.WithField("host", remote.Addr) logger.Info("delivering mail from peer using smarthost") @@ -223,30 +223,28 @@ func mailHandler(peer smtpd.Peer, env smtpd.Envelope) error { env.Data, ) if err != nil { - var smtpError smtpd.Error - switch err := err.(type) { case *textproto.Error: - smtpError = smtpd.Error{Code: err.Code, Message: err.Msg} - + smtpError = &smtpd.Error{Code: err.Code, Message: err.Msg} logger.WithFields(logrus.Fields{ "err_code": err.Code, "err_msg": err.Msg, }).Error("delivery failed") default: - smtpError = smtpd.Error{Code: 554, Message: "Forwarding failed"} - + smtpError = &smtpd.Error{Code: 554, Message: "Forwarding failed"} logger.WithError(err). Error("delivery failed") } - - return smtpError } - + } + if smtpError == nil { logger.Debug("delivery successful") + return nil + } else { + logger.Debug("do not direct send") + return *smtpError } - return nil } func generateUUID() string { diff --git a/ratelimiter.go b/ratelimiter.go new file mode 100644 index 00000000..93d2c453 --- /dev/null +++ b/ratelimiter.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "os" + "strings" + "sync" + "time" + + "github.com/maypok86/otter" +) + +var lock = &sync.Mutex{} + +type single struct { + context context.Context + cache otter.Cache[string, string] + r *Remote +} + +var remoteCache map[*Remote]*single = make(map[*Remote]*single) + +func getContext(r *Remote) *single { + + if remoteCache[r] == nil { + lock.Lock() + defer lock.Unlock() + if remoteCache[r] == nil { + remoteCache[r] = &single{} + + remoteCache[r].context = context.Background() + remoteCache[r].r = r + + cache, err := otter.MustBuilder[string, string](10_000). + CollectStats(). + Cost(func(key string, value string) uint32 { + return 1 + }).DeletionListener(func(key, value string, cause otter.DeletionCause) { + log.Infof("Evicted %s %s %v ", key, value, cause) + + parts := strings.Split(value, ";") + if len(parts) < 3 { + log.Info("Should have had at least three parts") + } else { + msg, err := os.ReadFile("/tmp/" + key + ".mail") + if err != nil { + log.Errorf("cannot read file %s", key+".mail") + + } else { + from := parts[1] + to := parts[2:] + SendMail(r, from, to, msg) + os.Remove("/tmp/" + key + ".mail") + } + } + }). + WithTTL(time.Minute). + Build() + + if err != nil { + panic(err) + } + remoteCache[r].cache = cache + } + } + return remoteCache[r] +} diff --git a/remotes.go b/remotes.go index 8e7ba80f..ed9853cd 100644 --- a/remotes.go +++ b/remotes.go @@ -4,16 +4,23 @@ import ( "fmt" "net/smtp" "net/url" + "strconv" + "strings" + "time" + + "github.com/sethvargo/go-limiter" + "github.com/sethvargo/go-limiter/memorystore" ) type Remote struct { - SkipVerify bool - Auth smtp.Auth - Scheme string - Hostname string - Port string - Addr string - Sender string + SkipVerify bool + Auth smtp.Auth + Scheme string + Hostname string + Port string + Addr string + Sender string + RateLimiter *limiter.Store } // ParseRemote creates a remote from a given url in the following format: @@ -79,5 +86,22 @@ func ParseRemote(remoteURL string) (*Remote, error) { r.Sender = u.Path[1:] } + if hasVal, rate := q.Has("rate"), q.Get("rate"); hasVal && strings.Contains(rate, "/") { + i, err := strconv.ParseInt(strings.Split(rate, "/")[0], 10, 32) + if err == nil { + t, err := time.ParseDuration(strings.Split(rate, "/")[1]) + log.Infof("Configuring rate limiter %v/%v", i, t) + if err == nil { + store, err := memorystore.New(&memorystore.Config{ + Tokens: uint64(i), + Interval: t, + }) + if err == nil { + r.RateLimiter = &store + } + } + } + } + return r, nil } diff --git a/send4mail.sh b/send4mail.sh new file mode 100644 index 00000000..e18a2690 --- /dev/null +++ b/send4mail.sh @@ -0,0 +1,6 @@ +#! /bin/bash +swaks --to recipient1@example.com --cc test@test.fr --from sender1@example.com --header "Subject: Test Email" --body "This is a test email sent using swaks." --server 127.0.0.1 --port 1025 +swaks --to recipient2@example.com --cc test@test.fr --from sender2@example.com --header "Subject: Test Email" --body "This is a test email sent using swaks." --server 127.0.0.1 --port 1025 +swaks --to recipient3@example.com --cc test@test.fr --from sender3@example.com --header "Subject: Test Email" --body "This is a test email sent using swaks." --server 127.0.0.1 --port 1025 +swaks --to recipient4@example.com --cc test@test.fr --from sender4@example.com --header "Subject: Test Email" --body "This is a test email sent using swaks." --server 127.0.0.1 --port 1025 + diff --git a/smtp.go b/smtp.go index d8bca149..798bbc44 100644 --- a/smtp.go +++ b/smtp.go @@ -26,7 +26,11 @@ import ( "net" "net/smtp" "net/textproto" + "os" "strings" + "time" + + "github.com/chrj/smtpd" ) // A Client represents a client connection to an SMTP server. @@ -320,7 +324,28 @@ var testHookStartTLS func(*tls.Config) // nil, except for tests // attachments (see the mime/multipart package), or other mail // functionality. Higher-level packages exist outside of the standard // library. + func SendMail(r *Remote, from string, to []string, msg []byte) error { + if r.RateLimiter != nil { + // Do the background in the main + tokens, remaining, _, ok, err := (*r.RateLimiter).Take(getContext(r).context, "") + log.Infof("Remaining %v tokens of %v", remaining, tokens) + + if err != nil || !ok { + // return smtpd.Error{Code: 452, Message: "Rate limit reached"} + theTime := time.Now() + filename := theTime.Format("2006-1-2-15-4-5") + ";" + from + ";" + strings.Join(to, ";") + filenameb64 := base64.URLEncoding.EncodeToString([]byte(filename)) + err := os.WriteFile("/tmp/"+filenameb64+".mail", msg, 0644) + getContext(r).cache.Set(filenameb64, filename) + if err != nil { + // handle error + } + return smtpd.Error{Code: 452, Message: "Rate limit reached"} + + } + log.Debugf("Remaining %v tokens of %v", remaining, tokens) + } if r.Sender != "" { from = r.Sender } diff --git a/smtprelay.ini b/smtprelay.ini index a6adf2c4..ca8826fe 100644 --- a/smtprelay.ini +++ b/smtprelay.ini @@ -8,20 +8,20 @@ ;logfile = ; Log format: default, plain (no timestamp), json -;log_format = default +log_format = default ; Log level: panic, fatal, error, warn, info, debug, trace -;log_level = info +log_level = info ; Hostname for this SMTP server -;hostname = localhost.localdomain +hostname = localhost.localdomain ; Welcome message for clients ;welcome_msg = ESMTP ready. ; Listen on the following addresses for incoming ; unencrypted connections. -;listen = 127.0.0.1:25 [::1]:25 +listen = 127.0.0.1:1025 ; STARTTLS and TLS are also supported but need a ; SSL certificate and key. @@ -37,7 +37,6 @@ ; Only use remotes where FROM EMail address in received ; EMail matches remote_sender. ;strict_sender = false - ; Socket timeout for read operations ; Duration string as sequence of decimal numbers, ; each with optional fraction and a unit suffix. @@ -126,5 +125,10 @@ ; Multiple remotes, space delimited ;remotes = smtp://127.0.0.1:1025 starttls://user:pass@smtp.mailgun.org:587 +; rate limit +; remotes = smtp://127.0.0.1:2525?rate=99/21m +remotes = smtp://127.0.0.1:2525?rate=1/1m smtp://127.0.0.1:2527?rate=6/1m + + ; Pipe messages to external command ;command = /usr/local/bin/script