precompute /users

dimkr · Aug 23, 2024 · 09b6d84 · 09b6d84
1 parent a69928f
commit 09b6d84
Show file tree

Hide file tree

Showing 21 changed files with 720 additions and 234 deletions.
diff --git a/README.md b/README.md
diff --git a/SETUP.md b/SETUP.md
@@ -106,7 +106,7 @@ If you have a graphical web browser and a Gemini client that configures itself a
 
 11. Register by creating a client certificate or clicking "Sign in" and use "View profile" to verify that your instance is able to "discover" users on other servers.
 
-Once a user is discovered, you can follow this user and your instance should start receiving new posts by this user. They should appear under your user's inbox ("My feed") and the user's profile.
+Once a user is discovered, you can follow this user and your instance should start receiving new posts by this user. They should appear under your user's inbox ("My feed") after a while (`FeedUpdateInterval`) and the user's profile.
 
 **If you don't see any posts, check tootik's output.**
 

diff --git a/cfg/cfg.go b/cfg/cfg.go
@@ -108,11 +108,14 @@ type Config struct {
 	FollowersSyncBatchSize int
 	FollowersSyncInterval  time.Duration
 
+	FeedUpdateInterval time.Duration
+
 	NotesTTL          time.Duration
 	InvisiblePostsTTL time.Duration
 	DeliveryTTL       time.Duration
 	SharesTTL         time.Duration
 	ActorTTL          time.Duration
+	FeedTTL           time.Duration
 }
 
 // FillDefaults replaces missing or invalid settings with defaults.
@@ -364,6 +367,10 @@ func (c *Config) FillDefaults() {
 		c.FollowersSyncInterval = time.Hour * 24 * 3
 	}
 
+	if c.FeedUpdateInterval <= 0 {
+		c.FeedUpdateInterval = time.Minute * 10
+	}
+
 	if c.NotesTTL <= 0 {
 		c.NotesTTL = time.Hour * 24 * 30
 	}
@@ -383,4 +390,8 @@ func (c *Config) FillDefaults() {
 	if c.ActorTTL <= 0 {
 		c.ActorTTL = time.Hour * 24 * 7
 	}
+
+	if c.FeedTTL <= 0 {
+		c.FeedTTL = time.Hour * 24 * 7
+	}
 }
diff --git a/cmd/tootik/main.go b/cmd/tootik/main.go
@@ -442,6 +442,15 @@ func main() {
 			Run(context.Context) error
 		}
 	}{
+		{
+			"feed",
+			cfg.FeedUpdateInterval,
+			&inbox.FeedUpdater{
+				Domain: *domain,
+				Config: &cfg,
+				DB:     db,
+			},
+		},
 		{
 			"poller",
 			pollResultsUpdateInterval,
@@ -495,10 +504,12 @@ func main() {
 
 			for {
 				log.Info("Running periodic job", "job", job.Name)
+				start := time.Now()
 				if err := job.Runner.Run(ctx); err != nil {
 					log.Error("Periodic job has failed", "job", job.Name, "error", err)
 					break
 				}
+				log.Info("Done running periodic job", "job", job.Name, "duration", time.Since(start).String())
 
 				select {
 				case <-ctx.Done():

diff --git a/data/garbage.go b/data/garbage.go
@@ -78,5 +78,9 @@ func (gc *GarbageCollector) Run(ctx context.Context) error {
 		return fmt.Errorf("failed to remove idle actors: %w", err)
 	}
 
+	if _, err := gc.DB.ExecContext(ctx, `delete from feed where inserted < ?`, now.Add(-gc.Config.FeedTTL).Unix()); err != nil {
+		return fmt.Errorf("failed to tri feed: %w", err)
+	}
+
 	return nil
 }
diff --git a/fed/resolve.go b/fed/resolve.go
@@ -140,11 +140,19 @@ func (r *Resolver) tryResolveOrCache(ctx context.Context, log *slog.Logger, db *
 }
 
 func deleteActor(ctx context.Context, log *slog.Logger, db *sql.DB, id string) {
-	if _, err := db.ExecContext(ctx, `delete from notesfts where id in (select id from notes where author = ?)`, id); err != nil {
+	if _, err := db.ExecContext(ctx, `delete from notesfts where exists (select 1 from notes where notes.author = ? and notesfts.id = notes.id)`, id); err != nil {
 		log.Warn("Failed to delete notes by actor", "id", id, "error", err)
 	}
 
-	if _, err := db.ExecContext(ctx, `delete from shares where by = $1 or note in (select id from notes where author = $1)`, id); err != nil {
+	if _, err := db.ExecContext(ctx, `delete from shares where by = $1 or exists (select 1 from notes where notes.author = ? and notes.id = shares.note)`, id); err != nil {
+		log.Warn("Failed to delete shares by actor", "id", id, "error", err)
+	}
+
+	if _, err := db.ExecContext(ctx, `delete from feed where sharer->>'$.id' = ?`, id); err != nil {
+		log.Warn("Failed to delete shares by actor", "id", id, "error", err)
+	}
+
+	if _, err := db.ExecContext(ctx, `delete from feed where author->>'$.id' = ?`, id); err != nil {
 		log.Warn("Failed to delete shares by actor", "id", id, "error", err)
 	}
 
@@ -327,7 +335,12 @@ func (r *Resolver) tryResolve(ctx context.Context, log *slog.Logger, db *sql.DB,
 		return nil, cachedActor, fmt.Errorf("%s does not match %s", actor.ID, profile)
 	}
 
-	if _, err := db.ExecContext(
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, cachedActor, fmt.Errorf("failed to cache %s: %w", actor.ID, err)
+	}
+
+	if _, err := tx.ExecContext(
 		ctx,
 		`INSERT INTO persons(id, actor, fetched) VALUES($1, $2, UNIXEPOCH()) ON CONFLICT(id) DO UPDATE SET actor = $2, updated = UNIXEPOCH()`,
 		actor.ID,
@@ -336,6 +349,28 @@ func (r *Resolver) tryResolve(ctx context.Context, log *slog.Logger, db *sql.DB,
 		return nil, cachedActor, fmt.Errorf("failed to cache %s: %w", actor.ID, err)
 	}
 
+	if _, err := tx.ExecContext(
+		ctx,
+		`UPDATE feed SET author = ? WHERE author->>'$.id' = ?`,
+		string(body),
+		actor.ID,
+	); err != nil {
+		return nil, cachedActor, fmt.Errorf("failed to cache %s: %w", actor.ID, err)
+	}
+
+	if _, err := tx.ExecContext(
+		ctx,
+		`UPDATE feed SET sharer = ? WHERE sharer->>'$.id' = ?`,
+		string(body),
+		actor.ID,
+	); err != nil {
+		return nil, cachedActor, fmt.Errorf("failed to cache %s: %w", actor.ID, err)
+	}
+
+	if err := tx.Commit(); err != nil {
+		return nil, cachedActor, fmt.Errorf("failed to cache %s: %w", actor.ID, err)
+	}
+
 	if actor.Published == nil && cachedActor != nil && cachedActor.Published != nil {
 		actor.Published = cachedActor.Published
 	} else if actor.Published == nil && (cachedActor == nil || cachedActor.Published == nil) {

diff --git a/front/mentions.go b/front/mentions.go
@@ -18,7 +18,6 @@ package front
 
 import (
 	"database/sql"
-	"time"
 
 	"github.com/dimkr/tootik/front/text"
 )
@@ -35,60 +34,19 @@ func (h *Handler) mentions(w text.Writer, r *request, args ...string) {
 		"📞 Mentions",
 		func(offset int) (*sql.Rows, error) {
 			return r.Query(`
-				select object, actor, sharer, inserted from
-				(
-					select notes.id, notes.object, persons.actor, notes.inserted, null as sharer from
-					follows
-					join
-					persons
-					on
-						persons.id = follows.followed
-					join
-					notes
-					on
-						notes.author = follows.followed and
-						(
-							$1 in (notes.cc0, notes.to0, notes.cc1, notes.to1, notes.cc2, notes.to2) or
-							(notes.to2 is not null and exists (select 1 from json_each(notes.object->'$.to') where value = $1)) or
-							(notes.cc2 is not null and exists (select 1 from json_each(notes.object->'$.cc') where value = $1))
-						)
-					where
-						follows.follower = $1 and
-						notes.inserted >= $2
-					union all
-					select notes.id, notes.object, authors.actor, shares.inserted, sharers.actor as sharer from
-					follows
-					join
-					shares
-					on
-						shares.by = follows.followed
-					join
-					notes
-					on
-						notes.id = shares.note and
-						(
-							$1 in (notes.cc0, notes.to0, notes.cc1, notes.to1, notes.cc2, notes.to2) or
-							(notes.to2 is not null and exists (select 1 from json_each(notes.object->'$.to') where value = $1)) or
-							(notes.cc2 is not null and exists (select 1 from json_each(notes.object->'$.cc') where value = $1))
-						)
-					join
-					persons authors
-					on
-						authors.id = notes.author
-					join
-					persons sharers
-					on
-						sharers.id = follows.followed
-					where
-						follows.follower = $1 and
-						shares.inserted >= $2
-				)
+				select note, author, sharer, inserted from
+				feed
+				where
+					follower = $1 and
+					(
+						exists (select 1 from json_each(note->'$.to') where value = $1) or
+						exists (select 1 from json_each(note->'$.cc') where value = $1)
+					)
 				order by
 					inserted desc
-				limit $3
-				offset $4`,
+				limit $2
+				offset $3`,
 				r.User.ID,
-				time.Now().Add(-time.Hour*24*7).Unix(),
 				h.Config.PostsPerPage,
 				offset,
 			)

diff --git a/front/users.go b/front/users.go
@@ -18,7 +18,6 @@ package front
 
 import (
 	"database/sql"
-	"time"
 
 	"github.com/dimkr/tootik/front/text"
 )
@@ -35,76 +34,15 @@ func (h *Handler) users(w text.Writer, r *request, args ...string) {
 		"📻 My Feed",
 		func(offset int) (*sql.Rows, error) {
 			return r.Query(`
-				select object, actor, sharer, inserted from
-				(
-					select id, object, actor, inserted, null as sharer from
-					(
-						select notes.id, notes.object, persons.actor, notes.inserted from
-						follows
-						join
-						persons
-						on
-							persons.id = follows.followed
-						join
-						notes
-						on
-							notes.author = follows.followed and
-							(
-								notes.public = 1 or
-								persons.actor->>'$.followers' in (notes.cc0, notes.to0, notes.cc1, notes.to1, notes.cc2, notes.to2) or
-								$1 in (notes.cc0, notes.to0, notes.cc1, notes.to1, notes.cc2, notes.to2) or
-								(notes.to2 is not null and exists (select 1 from json_each(notes.object->'$.to') where value = persons.actor->>'$.followers' or value = $1)) or
-								(notes.cc2 is not null and exists (select 1 from json_each(notes.object->'$.cc') where value = persons.actor->>'$.followers' or value = $1))
-							)
-						where
-							follows.follower = $1 and
-							notes.inserted >= $2
-						union
-						select notes.id, notes.object, authors.actor, notes.inserted from
-						notes myposts
-						join
-						notes
-						on
-							notes.object->>'$.inReplyTo' = myposts.id
-						join
-						persons authors
-						on
-							authors.id = notes.author
-						where
-							myposts.author = $1 and
-							notes.author != $1 and
-							notes.inserted >= $2
-					)
-					union all
-					select notes.id, notes.object, authors.actor, shares.inserted, sharers.actor as sharer from
-					follows
-					join
-					shares
-					on
-						shares.by = follows.followed
-					join
-					notes
-					on
-						notes.id = shares.note
-					join
-					persons authors
-					on
-						authors.id = notes.author
-					join
-					persons sharers
-					on
-						sharers.id = follows.followed
-					where
-						follows.follower = $1 and
-						shares.inserted >= $2 and
-						notes.public = 1
-				)
+				select note, author, sharer, inserted from
+				feed
+				where
+					follower = $1
 				order by
 					inserted desc
-				limit $3
-				offset $4`,
+				limit $2
+				offset $3`,
 				r.User.ID,
-				time.Now().Add(-time.Hour*24).Unix(),
 				h.Config.PostsPerPage,
 				offset,
 			)