From 17baf9a8830c92a669299649c111d420c655ff1e Mon Sep 17 00:00:00 2001
From: Mechiel Lukkien <mechiel@ueber.net>
Date: Sat, 7 Dec 2024 16:53:53 +0100
Subject: [PATCH] junk filter: fix adjusting word counts after train/untrain

after seeing some junk messages pass the filter, i investigated word counts in
junkfilter.db. i had seen suspicious counts that were just around powers of
two. did not make sense at the time. more investigating makes it clear: instead
of setting new word counts when updating the junk filter, we were adding the
new value to the current value (instead of just setting the new value). so the
counts got approximately doubled when being updated.

users should retrain the junk filter after this update using the "retrain"
subcommand.

this also adds logging for the hypothetical case where numbers would get
decreased below zero (which would wrap around due to uints).

and this fixes junk filter tests that were passing wrong parameters to
train/untrain...
---
 ctl.go              |  2 ++
 junk/filter.go      | 22 +++++++++++++++++-----
 junk/filter_test.go |  8 ++++----
 3 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/ctl.go b/ctl.go
index 0efeb2a0cb..81f73e4f46 100644
--- a/ctl.go
+++ b/ctl.go
@@ -1348,6 +1348,8 @@ func servectlcmd(ctx context.Context, ctl *ctl, shutdown func()) {
 			}
 		}()
 
+		// todo: can we retrain an account without holding a write lock? perhaps by writing a junkfilter to a new location, and staying informed of message changes while we go through all messages in the account?
+
 		acc.WithWLock(func() {
 			conf, _ := acc.Conf()
 			if conf.JunkFilter == nil {
diff --git a/junk/filter.go b/junk/filter.go
index 387ea493ab..d37e53c926 100644
--- a/junk/filter.go
+++ b/junk/filter.go
@@ -298,7 +298,7 @@ func (f *Filter) Save() error {
 			} else if err != nil {
 				return err
 			}
-			return tx.Update(&wordscore{w, wc.Ham + ham, wc.Spam + spam})
+			return tx.Update(&wordscore{w, ham, spam})
 		}
 		if err := update("-", f.hams, f.spams); err != nil {
 			return fmt.Errorf("storing total ham/spam message count: %s", err)
@@ -621,10 +621,16 @@ func (f *Filter) Untrain(ctx context.Context, ham bool, words map[string]struct{
 
 	// Modify the message count.
 	f.modified = true
+	var fv *uint32
 	if ham {
-		f.hams--
+		fv = &f.hams
 	} else {
-		f.spams--
+		fv = &f.spams
+	}
+	if *fv == 0 {
+		f.log.Error("attempt to decrease ham/spam message count while already zero", slog.Bool("ham", ham))
+	} else {
+		*fv -= 1
 	}
 
 	// Decrease the word counts.
@@ -633,10 +639,16 @@ func (f *Filter) Untrain(ctx context.Context, ham bool, words map[string]struct{
 		if !ok {
 			continue
 		}
+		var v *uint32
 		if ham {
-			c.Ham--
+			v = &c.Ham
+		} else {
+			v = &c.Spam
+		}
+		if *v == 0 {
+			f.log.Error("attempt to decrease ham/spam word count while already zero", slog.String("word", w), slog.Bool("ham", ham))
 		} else {
-			c.Spam--
+			*v -= 1
 		}
 		f.cache[w] = c
 		f.changed[w] = c
diff --git a/junk/filter_test.go b/junk/filter_test.go
index bb67c0f164..6aea0ef134 100644
--- a/junk/filter_test.go
+++ b/junk/filter_test.go
@@ -126,7 +126,7 @@ func TestFilter(t *testing.T) {
 	tcheck(t, err, "train spam message")
 	_, err = spamf.Seek(0, 0)
 	tcheck(t, err, "seek spam message")
-	err = f.TrainMessage(ctxbg, spamf, spamsize, true)
+	err = f.TrainMessage(ctxbg, spamf, spamsize, false)
 	tcheck(t, err, "train spam message")
 
 	if !f.modified {
@@ -166,16 +166,16 @@ func TestFilter(t *testing.T) {
 	tcheck(t, err, "untrain ham message")
 	_, err = hamf.Seek(0, 0)
 	tcheck(t, err, "seek ham message")
-	err = f.UntrainMessage(ctxbg, hamf, spamsize, true)
+	err = f.UntrainMessage(ctxbg, hamf, hamsize, true)
 	tcheck(t, err, "untrain ham message")
 
 	_, err = spamf.Seek(0, 0)
 	tcheck(t, err, "seek spam message")
-	err = f.UntrainMessage(ctxbg, spamf, spamsize, true)
+	err = f.UntrainMessage(ctxbg, spamf, spamsize, false)
 	tcheck(t, err, "untrain spam message")
 	_, err = spamf.Seek(0, 0)
 	tcheck(t, err, "seek spam message")
-	err = f.UntrainMessage(ctxbg, spamf, spamsize, true)
+	err = f.UntrainMessage(ctxbg, spamf, spamsize, false)
 	tcheck(t, err, "untrain spam message")
 
 	if !f.modified {