implement "requiretls", rfc 8689

with requiretls, the tls verification mode/rules for email deliveries can be
changed by the sender/submitter. in two ways:

1. "requiretls" smtp extension to always enforce verified tls (with mta-sts or
dnssec+dane), along the entire delivery path until delivery into the final
destination mailbox (so entire transport is verified-tls-protected).

2. "tls-required: no" message header, to ignore any tls and tls verification
errors even if the recipient domain has a policy that requires tls verification
(mta-sts and/or dnssec+dane), allowing delivery of non-sensitive messages in
case of misconfiguration/interoperability issues (at least useful for sending
tls reports).

we enable requiretls by default (only when tls is active), for smtp and
submission. it can be disabled through the config.

for each delivery attempt, we now store (per recipient domain, in the account
of the sender) whether the smtp server supports starttls and requiretls. this
support is shown (after having sent a first message) in the webmail when
sending a message (the previous 3 bars under the address input field are now 5
bars, the first for starttls support, the last for requiretls support). when
all recipient domains for a message are known to implement requiretls,
requiretls is automatically selected for sending (instead of "default" tls
behaviour). users can also select the "fallback to insecure" to add the
"tls-required: no" header.

new metrics are added for insight into requiretls errors and (some, not yet
all) cases where tls-required-no ignored a tls/verification error.

the admin can change the requiretls status for messages in the queue. so with
default delivery attempts, when verified tls is required by failing, an admin
could potentially change the field to "tls-required: no"-behaviour.

messages received (over smtp) with the requiretls option, get a comment added
to their Received header line, just before "id", after "with".

README.md

@@ -20,7 +20,7 @@ See Quickstart below to get started.
   ("localparts"), and in domain names (IDNA).
 - Automatic TLS with ACME, for use with Let's Encrypt and other CA's.
 - DANE and MTA-STS for inbound and outbound delivery over SMTP with STARTTLS,
-  with incoming TLSRPT reporting.
+  including REQUIRETLS and with incoming TLSRPT reporting.
 - Web admin interface that helps you set up your domains and accounts
   (instructions to create DNS records, configure
   SPF/DKIM/DMARC/TLSRPT/MTA-STS), for status information, managing

config/config.go

@@ -123,10 +123,14 @@ type Listener struct {
 	SMTPMaxMessageSize int64 `sconf:"optional" sconf-doc:"Maximum size in bytes for incoming and outgoing messages. Default is 100MB."`
 	SMTP               struct {
 		Enabled         bool
-		Port            int      `sconf:"optional" sconf-doc:"Default 25."`
-		NoSTARTTLS      bool     `sconf:"optional" sconf-doc:"Do not offer STARTTLS to secure the connection. Not recommended."`
-		RequireSTARTTLS bool     `sconf:"optional" sconf-doc:"Do not accept incoming messages if STARTTLS is not active. Can be used in combination with a strict MTA-STS policy. A remote SMTP server may not support TLS and may not be able to deliver messages."`
-		DNSBLs          []string `sconf:"optional" sconf-doc:"Addresses of DNS block lists for incoming messages. Block lists are only consulted for connections/messages without enough reputation to make an accept/reject decision. This prevents sending IPs of all communications to the block list provider. If any of the listed DNSBLs contains a requested IP address, the message is rejected as spam. The DNSBLs are checked for healthiness before use, at most once per 4 hours. Example DNSBLs: sbl.spamhaus.org, bl.spamcop.net. See https://www.spamhaus.org/sbl/ and https://www.spamcop.net/ for more information and terms of use."`
+		Port            int  `sconf:"optional" sconf-doc:"Default 25."`
+		NoSTARTTLS      bool `sconf:"optional" sconf-doc:"Do not offer STARTTLS to secure the connection. Not recommended."`
+		RequireSTARTTLS bool `sconf:"optional" sconf-doc:"Do not accept incoming messages if STARTTLS is not active. Can be used in combination with a strict MTA-STS policy. A remote SMTP server may not support TLS and may not be able to deliver messages."`
+		NoRequireTLS    bool `sconf:"optional" sconf-doc:"Do not announce the REQUIRETLS SMTP extension. Messages delivered using the REQUIRETLS extension should only be distributed onwards to servers also implementing the REQUIRETLS extension. In some situations, such as hosting mailing lists, this may not be feasible due to lack of support for the extension by mailing list subscribers."`
+		// Reoriginated messages (such as messages sent to mailing list subscribers) should
+		// keep REQUIRETLS. ../rfc/8689:412
+
+		DNSBLs []string `sconf:"optional" sconf-doc:"Addresses of DNS block lists for incoming messages. Block lists are only consulted for connections/messages without enough reputation to make an accept/reject decision. This prevents sending IPs of all communications to the block list provider. If any of the listed DNSBLs contains a requested IP address, the message is rejected as spam. The DNSBLs are checked for healthiness before use, at most once per 4 hours. Example DNSBLs: sbl.spamhaus.org, bl.spamcop.net. See https://www.spamhaus.org/sbl/ and https://www.spamcop.net/ for more information and terms of use."`
 
 		FirstTimeSenderDelay *time.Duration `sconf:"optional" sconf-doc:"Delay before accepting a message from a first-time sender for the destination account. Default: 15s."`
 

config/doc.go

@@ -184,6 +184,13 @@ describe-static" and "mox config describe-domains":
 				# TLS and may not be able to deliver messages. (optional)
 				RequireSTARTTLS: false
 
+				# Do not announce the REQUIRETLS SMTP extension. Messages delivered using the
+				# REQUIRETLS extension should only be distributed onwards to servers also
+				# implementing the REQUIRETLS extension. In some situations, such as hosting
+				# mailing lists, this may not be feasible due to lack of support for the extension
+				# by mailing list subscribers. (optional)
+				NoRequireTLS: false
+
 				# Addresses of DNS block lists for incoming messages. Block lists are only
 				# consulted for connections/messages without enough reputation to make an
 				# accept/reject decision. This prevents sending IPs of all communications to the

dane/dane.go

@@ -297,7 +297,7 @@ func Dial(ctx context.Context, resolver dns.Resolver, network, address string, a
 // verified, if any.
 func TLSClientConfig(log *mlog.Log, records []adns.TLSA, allowedHost dns.Domain, moreAllowedHosts []dns.Domain, verifiedRecord *adns.TLSA) tls.Config {
 	return tls.Config{
-		ServerName:         allowedHost.ASCII,
+		ServerName:         allowedHost.ASCII, // For SNI.
 		InsecureSkipVerify: true,
 		VerifyConnection: func(cs tls.ConnectionState) error {
 			verified, record, err := Verify(log, records, cs, allowedHost, moreAllowedHosts)

gentestdata.go

@@ -233,7 +233,7 @@ Accounts:
 	const qmsg = "From: <[email protected]>\r\nTo: <[email protected]>\r\nSubject: test\r\n\r\nthe message...\r\n"
 	_, err = fmt.Fprint(mf, qmsg)
 	xcheckf(err, "writing message")
-	_, err = queue.Add(ctxbg, log, "test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, mf, nil)
+	_, err = queue.Add(ctxbg, log, "test0", mailfrom, rcptto, false, false, int64(len(qmsg)), "<test@localhost>", prefix, mf, nil, nil)
 	xcheckf(err, "enqueue message")
 
 	// Create three accounts.

integration_test.go

@@ -131,7 +131,7 @@ This is the message.
 		auth := []sasl.Client{sasl.NewClientPlain(mailfrom, password)}
 		c, err := smtpclient.New(mox.Context, xlog, conn, smtpclient.TLSSkip, ourHostname, dns.Domain{ASCII: desthost}, auth, nil, nil, nil)
 		tcheck(t, err, "smtp hello")
-		err = c.Deliver(mox.Context, mailfrom, rcptto, int64(len(msg)), strings.NewReader(msg), false, false)
+		err = c.Deliver(mox.Context, mailfrom, rcptto, int64(len(msg)), strings.NewReader(msg), false, false, false)
 		tcheck(t, err, "deliver with smtp")
 		err = c.Close()
 		tcheck(t, err, "close smtpclient")

main.go

@@ -130,6 +130,7 @@ var commands = []struct {
 	{"cid", cmdCid},
 	{"clientconfig", cmdClientConfig},
 	{"deliver", cmdDeliver},
+	// todo: turn cmdDANEDialmx into a regular "dialmx" command that follows mta-sts policy, with options to require dane, mta-sts or requiretls. the code will be similar to queue/direct.go
 	{"dane dial", cmdDANEDial},
 	{"dane dialmx", cmdDANEDialmx},
 	{"dane makerecord", cmdDANEMakeRecord},