From 5bebe31765e05803c42d561ad1ecaf3ce4128860 Mon Sep 17 00:00:00 2001
From: Mikkel Denker <mikkel@stract.com>
Date: Tue, 19 Mar 2024 16:21:02 +0100
Subject: [PATCH] refactor data that is re-used across fields for a particular
 page during indexing into an 'FnCache'

---
 Cargo.lock                                    |  13 +
 Cargo.toml                                    |   1 +
 assets/licenses.html                          |   3 +-
 crates/core/Cargo.toml                        |   1 +
 crates/core/src/mapreduce/dht/mod.rs          |   8 +-
 crates/core/src/mapreduce/dht/network/mod.rs  |  20 +-
 crates/core/src/mapreduce/dht/network/raft.rs |  28 +-
 crates/core/src/query/parser/mod.rs           |   5 +-
 crates/core/src/webpage/html/fn_cache.rs      | 136 +++++++
 crates/core/src/webpage/html/into_tantivy.rs  | 356 ++++++++++--------
 crates/core/src/webpage/html/mod.rs           |   1 +
 11 files changed, 388 insertions(+), 184 deletions(-)
 create mode 100644 crates/core/src/webpage/html/fn_cache.rs

diff --git a/Cargo.lock b/Cargo.lock
index b22de01f..a844080c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1373,6 +1373,18 @@ dependencies = [
  "syn 2.0.52",
 ]
 
+[[package]]
+name = "enum_dispatch"
+version = "0.3.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f33313078bb8d4d05a2733a94ac4c2d8a0df9a2b84424ebf4f33bfc224a890e"
+dependencies = [
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
 [[package]]
 name = "equivalent"
 version = "1.0.1"
@@ -4536,6 +4548,7 @@ dependencies = [
  "csv",
  "dashmap",
  "encoding_rs",
+ "enum_dispatch",
  "eventsource-stream",
  "fend-core",
  "flate2",
diff --git a/Cargo.toml b/Cargo.toml
index f7c975af..a5209b61 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,6 +40,7 @@ crossbeam-channel = "0.5.6"
 csv = "1.1.6"
 dashmap = { version = "5.4.0", features = ["rayon"] }
 encoding_rs = "0.8.31"
+enum_dispatch = "0.3.12"
 eventsource-stream = "0.2.3"
 fend-core = "1.2.2"
 flate2 = "1.0.28"
diff --git a/assets/licenses.html b/assets/licenses.html
index f9d0aa04..e51ee66c 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -44,7 +44,7 @@ <h1>Third Party Licenses</h1>
     
         <h2>Overview of licenses:</h2>
         <ul class="licenses-overview">
-            <li><a href="#Apache-2.0">Apache License 2.0</a> (394)</li>
+            <li><a href="#Apache-2.0">Apache License 2.0</a> (395)</li>
             <li><a href="#MIT">MIT License</a> (180)</li>
             <li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
             <li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (8)</li>
@@ -9889,6 +9889,7 @@ <h4>Used by:</h4>
                     <li><a href=" https://github.com/zrzka/anes-rs ">anes 0.1.6</a></li>
                     <li><a href=" https://github.com/huggingface/candle ">candle-nn 0.3.3</a></li>
                     <li><a href=" https://github.com/huggingface/candle ">candle-transformers 0.3.3</a></li>
+                    <li><a href=" https://gitlab.com/antonok/enum_dispatch ">enum_dispatch 0.3.12</a></li>
                     <li><a href=" https://github.com/jpopesculian/eventsource-stream ">eventsource-stream 0.2.3</a></li>
                     <li><a href=" https://github.com/cbreeden/fxhash ">fxhash 0.2.1</a></li>
                     <li><a href=" https://github.com/starkat99/half-rs ">half 2.4.0</a></li>
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 520c2940..ca7d80cb 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -45,6 +45,7 @@ crossbeam-channel = { workspace = true }
 csv = { workspace = true }
 dashmap = { workspace = true }
 encoding_rs = { workspace = true }
+enum_dispatch = { workspace = true }
 eventsource-stream = { workspace = true }
 fend-core = { workspace = true }
 flate2 = { workspace = true }
diff --git a/crates/core/src/mapreduce/dht/mod.rs b/crates/core/src/mapreduce/dht/mod.rs
index 08b8af32..6667c906 100644
--- a/crates/core/src/mapreduce/dht/mod.rs
+++ b/crates/core/src/mapreduce/dht/mod.rs
@@ -73,18 +73,18 @@ macro_rules! raft_sonic_request_response {
         #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
         pub enum Response {
             $(
-                $req(<$req as crate::distributed::sonic::service::Message<$service>>::Response),
+                $req(<$req as $crate::distributed::sonic::service::Message<$service>>::Response),
             )*
             Empty,
         }
 
         $(
-        impl TryFrom<Response> for <$req as crate::distributed::sonic::service::Message<$service>>::Response {
-            type Error = crate::distributed::sonic::Error;
+        impl TryFrom<Response> for <$req as $crate::distributed::sonic::service::Message<$service>>::Response {
+            type Error = $crate::distributed::sonic::Error;
             fn try_from(res: Response) -> Result<Self, Self::Error> {
                 match res {
                     Response::$req(res) => Ok(res),
-                    _ => Err(crate::distributed::sonic::Error::Application(anyhow::anyhow!("Invalid response for request from Raft"))),
+                    _ => Err($crate::distributed::sonic::Error::Application(anyhow::anyhow!("Invalid response for request from Raft"))),
                 }
             }
         }
diff --git a/crates/core/src/mapreduce/dht/network/mod.rs b/crates/core/src/mapreduce/dht/network/mod.rs
index b6efc5a1..e3c1d01e 100644
--- a/crates/core/src/mapreduce/dht/network/mod.rs
+++ b/crates/core/src/mapreduce/dht/network/mod.rs
@@ -39,34 +39,34 @@ impl RaftNetworkFactory<TypeConfig> for Network {
     }
 }
 
-pub type AppendEntriesRequest = openraft::raft::AppendEntriesRequest<TypeConfig>;
+pub type AppendEntries = openraft::raft::AppendEntriesRequest<TypeConfig>;
 pub type AppendEntriesResponse = openraft::raft::AppendEntriesResponse<NodeId>;
 
-pub type InstallSnapshotRequest = openraft::raft::InstallSnapshotRequest<TypeConfig>;
+pub type InstallSnapshot = openraft::raft::InstallSnapshotRequest<TypeConfig>;
 pub type InstallSnapshotResponse = openraft::raft::InstallSnapshotResponse<NodeId>;
 
-pub type VoteRequest = openraft::raft::VoteRequest<NodeId>;
+pub type Vote = openraft::raft::VoteRequest<NodeId>;
 pub type VoteResponse = openraft::raft::VoteResponse<NodeId>;
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
-pub struct AddLearnerRequest {
+pub struct AddLearner {
     pub id: NodeId,
     pub addr: SocketAddr,
 }
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
-pub struct AddNodesRequest {
+pub struct AddNodes {
     members: BTreeMap<NodeId, BasicNode>,
 }
 
 sonic_service!(
     Server,
     [
-        AppendEntriesRequest,
-        InstallSnapshotRequest,
-        VoteRequest,
-        AddLearnerRequest,
-        AddNodesRequest,
+        AppendEntries,
+        InstallSnapshot,
+        Vote,
+        AddLearner,
+        AddNodes,
         Get,
         Set,
     ]
diff --git a/crates/core/src/mapreduce/dht/network/raft.rs b/crates/core/src/mapreduce/dht/network/raft.rs
index bfac4b56..0e1297f4 100644
--- a/crates/core/src/mapreduce/dht/network/raft.rs
+++ b/crates/core/src/mapreduce/dht/network/raft.rs
@@ -37,11 +37,11 @@ use crate::{
 };
 
 use super::{
-    AddLearnerRequest, AddNodesRequest, AppendEntriesRequest, AppendEntriesResponse,
-    InstallSnapshotRequest, InstallSnapshotResponse, Server, VoteRequest, VoteResponse,
+    AddLearner, AddNodes, AppendEntries, AppendEntriesResponse, InstallSnapshot,
+    InstallSnapshotResponse, Server, Vote, VoteResponse,
 };
 
-impl sonic::service::Message<Server> for AppendEntriesRequest {
+impl sonic::service::Message<Server> for AppendEntries {
     type Response = Result<AppendEntriesResponse, RaftError<NodeId>>;
 
     async fn handle(self, server: &Server) -> Self::Response {
@@ -50,7 +50,7 @@ impl sonic::service::Message<Server> for AppendEntriesRequest {
     }
 }
 
-impl sonic::service::Message<Server> for InstallSnapshotRequest {
+impl sonic::service::Message<Server> for InstallSnapshot {
     type Response = Result<InstallSnapshotResponse, RaftError<NodeId, InstallSnapshotError>>;
 
     async fn handle(self, server: &Server) -> Self::Response {
@@ -59,7 +59,7 @@ impl sonic::service::Message<Server> for InstallSnapshotRequest {
     }
 }
 
-impl sonic::service::Message<Server> for VoteRequest {
+impl sonic::service::Message<Server> for Vote {
     type Response = Result<VoteResponse, RaftError<NodeId>>;
 
     async fn handle(self, server: &Server) -> Self::Response {
@@ -68,7 +68,7 @@ impl sonic::service::Message<Server> for VoteRequest {
     }
 }
 
-impl sonic::service::Message<Server> for AddLearnerRequest {
+impl sonic::service::Message<Server> for AddLearner {
     type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;
 
     async fn handle(self, server: &Server) -> Self::Response {
@@ -93,7 +93,7 @@ impl sonic::service::Message<Server> for AddLearnerRequest {
     }
 }
 
-impl sonic::service::Message<Server> for AddNodesRequest {
+impl sonic::service::Message<Server> for AddNodes {
     type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;
 
     async fn handle(self, server: &Server) -> Self::Response {
@@ -160,7 +160,7 @@ impl RemoteClient {
     }
 
     async fn add_learner(&self, id: NodeId, addr: SocketAddr) -> Result<()> {
-        let rpc = AddLearnerRequest { id, addr };
+        let rpc = AddLearner { id, addr };
         let retry = ExponentialBackoff::from_millis(500)
             .with_limit(Duration::from_secs(60))
             .take(5);
@@ -220,7 +220,7 @@ impl RemoteClient {
     }
 
     async fn add_nodes(&self, members: BTreeMap<NodeId, BasicNode>) -> Result<()> {
-        let rpc = AddNodesRequest { members };
+        let rpc = AddNodes { members };
         let retry = ExponentialBackoff::from_millis(500).with_limit(Duration::from_secs(10));
 
         for backoff in retry {
@@ -293,7 +293,7 @@ impl RemoteClient {
 impl RaftNetwork<TypeConfig> for RemoteClient {
     async fn append_entries(
         &mut self,
-        rpc: AppendEntriesRequest,
+        rpc: AppendEntries,
         option: RPCOption,
     ) -> Result<AppendEntriesResponse, RPCError> {
         self.send_raft_rpc(rpc, option).await?.map_err(|e| {
@@ -308,7 +308,7 @@ impl RaftNetwork<TypeConfig> for RemoteClient {
 
     async fn install_snapshot(
         &mut self,
-        rpc: InstallSnapshotRequest,
+        rpc: InstallSnapshot,
         option: RPCOption,
     ) -> Result<InstallSnapshotResponse, RPCError<InstallSnapshotError>> {
         self.send_raft_rpc(rpc, option).await?.map_err(|e| {
@@ -321,11 +321,7 @@ impl RaftNetwork<TypeConfig> for RemoteClient {
         })
     }
 
-    async fn vote(
-        &mut self,
-        rpc: VoteRequest,
-        option: RPCOption,
-    ) -> Result<VoteResponse, RPCError> {
+    async fn vote(&mut self, rpc: Vote, option: RPCOption) -> Result<VoteResponse, RPCError> {
         self.send_raft_rpc(rpc, option).await?.map_err(|e| {
             openraft::error::RemoteError {
                 target: self.target,
diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rs
index 8968ea94..0e696cbc 100644
--- a/crates/core/src/query/parser/mod.rs
+++ b/crates/core/src/query/parser/mod.rs
@@ -93,8 +93,9 @@ fn single_bang(input: &str, pref: char) -> nom::IResult<&str, Term> {
 
 fn bang(input: &str) -> nom::IResult<&str, Term> {
     for pref in BANG_PREFIXES.iter() {
-        let (input, output) = single_bang(input, *pref)?;
-        return Ok((input, output));
+        if let Ok((input, output)) = single_bang(input, *pref) {
+            return Ok((input, output));
+        }
     }
 
     Err(nom::Err::Error(nom::error::Error::new(
diff --git a/crates/core/src/webpage/html/fn_cache.rs b/crates/core/src/webpage/html/fn_cache.rs
new file mode 100644
index 00000000..fee3abae
--- /dev/null
+++ b/crates/core/src/webpage/html/fn_cache.rs
@@ -0,0 +1,136 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{webpage::schema_org, Result};
+use tantivy::tokenizer::PreTokenizedString;
+
+use super::{find_recipe_first_ingredient_tag_id, Html};
+
+#[macro_export]
+macro_rules! cache {
+    ($($fn:ident -> $res:ty),*$(,)?) => {
+        /// Dynamically compute the different webpage functions
+        /// and cache the results for subsequent calls.
+        ///
+        /// Used during indexing as some of the fields use
+        /// the same data from the webpage and we don't want to
+        /// recompute the same data multiple times.
+        pub struct FnCache<'a> {
+            html: &'a Html,
+            first_ingredient_tag_id: Option<String>,
+            schema_json: Option<String>,
+            pretokenized_schema_json: Option<PreTokenizedString>,
+            $($fn: Option<$res>,)*
+        }
+
+        impl<'a> FnCache<'a> {
+            /// Create a new instance of the IndexingCacher
+            pub fn new(html: &'a Html) -> Self {
+                Self {
+                    html,
+                    first_ingredient_tag_id: None,
+                    schema_json: None,
+                    pretokenized_schema_json: None,
+                    $($fn: None,)*
+                }
+            }
+
+            $(
+                /// Compute $fn from webpage and cache the result
+                pub fn $fn(&mut self) -> &$res {
+                    if self.$fn.is_none() {
+                        self.$fn = Some(self.html.$fn());
+                    }
+
+                    self.$fn.as_ref().unwrap()
+                }
+            )*
+        }
+    };
+}
+
+cache! {
+    pretokenize_title -> Result<PreTokenizedString>,
+    pretokenize_all_text -> Result<PreTokenizedString>,
+    pretokenize_clean_text -> PreTokenizedString,
+    pretokenize_url -> PreTokenizedString,
+    pretokenize_url_for_site_operator -> PreTokenizedString,
+    pretokenize_domain -> PreTokenizedString,
+    pretokenize_site -> PreTokenizedString,
+    pretokenize_description -> PreTokenizedString,
+    pretokenize_microformats -> PreTokenizedString,
+    domain_name -> String,
+    schema_org -> Vec<schema_org::Item>,
+    site_hash -> [u64; 2],
+    url_without_query_hash -> [u64; 2],
+    url_hash -> [u64; 2],
+    url_without_tld_hash -> [u64; 2],
+    domain_hash -> [u64; 2],
+    title_hash -> [u64; 2],
+}
+
+/// Some manual implementations so we can use previously cached data
+/// to compute the next field.
+impl<'a> FnCache<'a> {
+    pub fn first_ingredient_tag_id(&mut self) -> Option<&String> {
+        if self.first_ingredient_tag_id.is_none() {
+            let root = self.html.root.clone(); // Node is just a NodeRef, so it's cheap to clone
+
+            self.first_ingredient_tag_id =
+                find_recipe_first_ingredient_tag_id(self.schema_org().as_slice(), &root);
+        }
+
+        self.first_ingredient_tag_id.as_ref()
+    }
+
+    pub fn schema_json(&mut self) -> &String {
+        if self.schema_json.is_none() {
+            self.schema_json = Some(serde_json::to_string(self.schema_org()).unwrap());
+        }
+
+        self.schema_json.as_ref().unwrap()
+    }
+
+    pub fn pretokenized_schema_json(&mut self) -> &PreTokenizedString {
+        if self.pretokenized_schema_json.is_none() {
+            self.pretokenized_schema_json =
+                match schema_org::flattened_json(self.schema_org().clone()) {
+                    Ok(mut f) => {
+                        let mut tokens = Vec::new();
+
+                        {
+                            let mut stream = f.token_stream();
+
+                            while let Some(token) = stream.next() {
+                                tokens.push(token.clone());
+                            }
+                        }
+
+                        Some(PreTokenizedString {
+                            text: f.text().to_string(),
+                            tokens,
+                        })
+                    }
+                    Err(_) => Some(PreTokenizedString {
+                        text: String::new(),
+                        tokens: Vec::new(),
+                    }),
+                };
+        }
+
+        self.pretokenized_schema_json.as_ref().unwrap()
+    }
+}
diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rs
index 3263e75f..6ea2fbc0 100644
--- a/crates/core/src/webpage/html/into_tantivy.rs
+++ b/crates/core/src/webpage/html/into_tantivy.rs
@@ -1,5 +1,5 @@
 // Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as
@@ -29,12 +29,12 @@ use tantivy::{
 };
 use whatlang::Lang;
 
-use super::{find_recipe_first_ingredient_tag_id, schema_org, Html};
+use super::{fn_cache::FnCache, Html};
 
 use crate::schema::{Field, FLOAT_SCALING};
 
 impl Html {
-    fn pretokenize_title(&self) -> Result<PreTokenizedString> {
+    pub fn pretokenize_title(&self) -> Result<PreTokenizedString> {
         let title = self.title();
 
         if title.is_none() {
@@ -45,7 +45,7 @@ impl Html {
         Ok(self.pretokenize_string(title, TextField::Title))
     }
 
-    fn pretokenize_all_text(&self) -> Result<PreTokenizedString> {
+    pub fn pretokenize_all_text(&self) -> Result<PreTokenizedString> {
         let all_text = self.all_text();
 
         if all_text.is_none() {
@@ -56,35 +56,42 @@ impl Html {
         Ok(self.pretokenize_string(all_text, TextField::AllBody))
     }
 
-    fn pretokenize_clean_text(&self) -> PreTokenizedString {
+    pub fn pretokenize_clean_text(&self) -> PreTokenizedString {
         let clean_text = self.clean_text().cloned().unwrap_or_default();
         self.pretokenize_string(clean_text, TextField::CleanBody)
     }
 
-    fn pretokenize_url(&self) -> PreTokenizedString {
+    pub fn pretokenize_url(&self) -> PreTokenizedString {
         let url = self.url().to_string();
         self.pretokenize_string(url, TextField::Url)
     }
 
-    fn pretokenize_domain(&self) -> PreTokenizedString {
+    pub fn pretokenize_url_for_site_operator(&self) -> PreTokenizedString {
+        self.pretokenize_string_with(
+            self.url().to_string(),
+            tokenizer::Tokenizer::SiteOperator(tokenizer::SiteOperatorUrlTokenizer),
+        )
+    }
+
+    pub fn pretokenize_domain(&self) -> PreTokenizedString {
         let domain = self.url().root_domain().unwrap_or_default().to_string();
 
         self.pretokenize_string(domain, TextField::Domain)
     }
 
-    fn pretokenize_site(&self) -> PreTokenizedString {
+    pub fn pretokenize_site(&self) -> PreTokenizedString {
         let site = self.url().normalized_host().unwrap_or_default().to_string();
 
         self.pretokenize_string(site, TextField::SiteWithout)
     }
 
-    fn pretokenize_description(&self) -> PreTokenizedString {
+    pub fn pretokenize_description(&self) -> PreTokenizedString {
         let text = self.description().unwrap_or_default();
 
         self.pretokenize_string(text, TextField::Description)
     }
 
-    fn pretokenize_microformats(&self) -> PreTokenizedString {
+    pub fn pretokenize_microformats(&self) -> PreTokenizedString {
         let mut text = String::new();
 
         for microformat in self.microformats().iter() {
@@ -118,6 +125,17 @@ impl Html {
         PreTokenizedString { text, tokens }
     }
 
+    pub fn domain_name(&self) -> String {
+        let domain = self.url().domain().unwrap_or_default();
+        self.url()
+            .root_domain()
+            .unwrap_or_default()
+            .find('.')
+            .map(|index| &domain[..ceil_char_boundary(&domain, index).min(domain.len())])
+            .unwrap_or_default()
+            .to_string()
+    }
+
     pub fn keywords(&self, rake: &RakeModel) -> Vec<String> {
         self.clean_text()
             .map(|text| {
@@ -129,70 +147,18 @@ impl Html {
             .unwrap_or_default()
     }
 
-    pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
-        let mut doc = TantivyDocument::new();
-
-        let title = self.pretokenize_title()?;
-        let all_text = self.pretokenize_all_text()?;
-        let clean_text = self.pretokenize_clean_text();
-        let url = self.pretokenize_url();
-        let domain = self.pretokenize_domain();
-        let site = self.pretokenize_site();
-        let description = self.pretokenize_description();
-        let microformats = self.pretokenize_microformats();
-        let url_for_site_operator = self.pretokenize_string_with(
-            self.url().to_string(),
-            tokenizer::Tokenizer::SiteOperator(tokenizer::SiteOperatorUrlTokenizer),
-        );
-
-        let domain_name = self
-            .url()
-            .root_domain()
-            .unwrap_or_default()
-            .find('.')
-            .map(|index| {
-                &domain.text[..ceil_char_boundary(&domain.text, index).min(domain.text.len())]
-            })
-            .unwrap_or_default()
-            .to_string();
-
-        let schemas: Vec<_> = self.schema_org();
-        let first_ingredient_tag_id =
-            find_recipe_first_ingredient_tag_id(&schemas, &self.root).unwrap_or_default();
-
-        let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();
-
-        let pretokenized_schema_json = match schema_org::flattened_json(schemas) {
-            Ok(mut f) => {
-                let mut tokens = Vec::new();
-
-                {
-                    let mut stream = f.token_stream();
-
-                    while let Some(token) = stream.next() {
-                        tokens.push(token.clone());
-                    }
-                }
-
-                PreTokenizedString {
-                    text: f.text().to_string(),
-                    tokens,
-                }
-            }
-            Err(_) => PreTokenizedString {
-                text: String::new(),
-                tokens: Vec::new(),
-            },
-        };
-
-        let site_hash = split_u128(hash(self.url().normalized_host().unwrap_or_default()).0);
+    pub fn site_hash(&self) -> [u64; 2] {
+        split_u128(hash(self.url().normalized_host().unwrap_or_default()).0)
+    }
 
+    pub fn url_without_query_hash(&self) -> [u64; 2] {
         let mut url_without_query = self.url().clone();
         url_without_query.set_query(None);
 
-        let url_without_query_hash = split_u128(hash(url_without_query.as_str()).0);
-        let url_hash = split_u128(hash(self.url().as_str()).0);
+        split_u128(hash(url_without_query.as_str()).0)
+    }
 
+    pub fn url_without_tld_hash(&self) -> [u64; 2] {
         let tld = self.url().tld().unwrap_or_default();
         let url_without_tld = self
             .url()
@@ -205,10 +171,24 @@ impl Html {
             + "?"
             + self.url().query().unwrap_or_default();
 
-        let url_without_tld_hash = split_u128(hash(url_without_tld).0);
+        split_u128(hash(url_without_tld).0)
+    }
 
-        let domain_hash = split_u128(hash(self.url().root_domain().unwrap_or_default()).0);
-        let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);
+    pub fn url_hash(&self) -> [u64; 2] {
+        split_u128(hash(self.url().as_str()).0)
+    }
+
+    pub fn domain_hash(&self) -> [u64; 2] {
+        split_u128(hash(self.url().root_domain().unwrap_or_default()).0)
+    }
+
+    pub fn title_hash(&self) -> [u64; 2] {
+        split_u128(hash(self.title().unwrap_or_default()).0)
+    }
+
+    pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
+        let mut doc = TantivyDocument::new();
+        let mut cache = FnCache::new(self);
 
         for field in schema
             .fields()
@@ -219,10 +199,20 @@ impl Html {
                 .unwrap_or_else(|_| panic!("Unknown field: {}", field.name()));
 
             match field {
-                Field::Text(TextField::Title) => {
-                    doc.add_pre_tokenized_text(tantivy_field, title.clone())
-                }
+                Field::Text(TextField::Title) => doc.add_pre_tokenized_text(
+                    tantivy_field,
+                    cache
+                        .pretokenize_title()
+                        .as_ref()
+                        .map(Clone::clone)
+                        .map_err(|e| anyhow::anyhow!("{}", e))?,
+                ),
                 Field::Text(TextField::StemmedTitle) => {
+                    let title = cache
+                        .pretokenize_title()
+                        .as_ref()
+                        .map(Clone::clone)
+                        .map_err(|e| anyhow::anyhow!("{}", e))?;
                     let mut tokens = title.tokens.clone();
                     stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
 
@@ -234,10 +224,10 @@ impl Html {
                         },
                     );
                 }
-                Field::Text(TextField::CleanBody) => {
-                    doc.add_pre_tokenized_text(tantivy_field, clean_text.clone())
-                }
+                Field::Text(TextField::CleanBody) => doc
+                    .add_pre_tokenized_text(tantivy_field, cache.pretokenize_clean_text().clone()),
                 Field::Text(TextField::StemmedCleanBody) => {
+                    let clean_text = cache.pretokenize_clean_text();
                     let mut tokens = clean_text.tokens.clone();
                     stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
 
@@ -262,20 +252,33 @@ impl Html {
                     );
                 }
                 Field::Text(TextField::TitleBigrams) => {
+                    let title = cache
+                        .pretokenize_title()
+                        .as_ref()
+                        .map(Clone::clone)
+                        .map_err(|e| anyhow::anyhow!("{}", e))?;
                     doc.add_text(tantivy_field, title.text.clone());
                 }
                 Field::Text(TextField::TitleTrigrams) => {
+                    let title = cache
+                        .pretokenize_title()
+                        .as_ref()
+                        .map(Clone::clone)
+                        .map_err(|e| anyhow::anyhow!("{}", e))?;
                     doc.add_text(tantivy_field, title.text.clone());
                 }
                 Field::Text(TextField::Description) => {
+                    let description = cache.pretokenize_description();
                     doc.add_pre_tokenized_text(tantivy_field, description.clone());
                 }
                 Field::Text(TextField::Url) => {
+                    let url = cache.pretokenize_url();
                     doc.add_pre_tokenized_text(tantivy_field, url.clone())
                 }
-                Field::Text(TextField::UrlForSiteOperator) => {
-                    doc.add_pre_tokenized_text(tantivy_field, url_for_site_operator.clone())
-                }
+                Field::Text(TextField::UrlForSiteOperator) => doc.add_pre_tokenized_text(
+                    tantivy_field,
+                    cache.pretokenize_url_for_site_operator().clone(),
+                ),
                 Field::Text(TextField::UrlNoTokenizer) => {
                     let url = self.url().to_string();
 
@@ -294,25 +297,31 @@ impl Html {
                     );
                 }
                 Field::Text(TextField::SiteWithout) => {
-                    doc.add_pre_tokenized_text(tantivy_field, site.clone())
+                    doc.add_pre_tokenized_text(tantivy_field, cache.pretokenize_site().clone())
                 }
                 Field::Text(TextField::Domain) => {
-                    doc.add_pre_tokenized_text(tantivy_field, domain.clone())
+                    doc.add_pre_tokenized_text(tantivy_field, cache.pretokenize_domain().clone())
                 }
-                Field::Text(TextField::SiteNoTokenizer) => doc.add_pre_tokenized_text(
-                    tantivy_field,
-                    PreTokenizedString {
-                        text: site.text.clone(),
-                        tokens: vec![tantivy::tokenizer::Token {
-                            offset_from: 0,
-                            offset_to: site.text.len(),
-                            position: 0,
+                Field::Text(TextField::SiteNoTokenizer) => {
+                    let site = cache.pretokenize_site();
+
+                    doc.add_pre_tokenized_text(
+                        tantivy_field,
+                        PreTokenizedString {
                             text: site.text.clone(),
-                            position_length: 1,
-                        }],
-                    },
-                ),
+                            tokens: vec![tantivy::tokenizer::Token {
+                                offset_from: 0,
+                                offset_to: site.text.len(),
+                                position: 0,
+                                text: site.text.clone(),
+                                position_length: 1,
+                            }],
+                        },
+                    )
+                }
                 Field::Text(TextField::SiteIfHomepageNoTokenizer) => {
+                    let site = cache.pretokenize_site();
+
                     if self.is_homepage() {
                         doc.add_pre_tokenized_text(
                             tantivy_field,
@@ -331,27 +340,38 @@ impl Html {
                         doc.add_text(tantivy_field, "");
                     }
                 }
-                Field::Text(TextField::DomainNoTokenizer) => doc.add_pre_tokenized_text(
-                    tantivy_field,
-                    PreTokenizedString {
-                        text: domain.text.clone(),
-                        tokens: vec![tantivy::tokenizer::Token {
-                            offset_from: 0,
-                            offset_to: domain.text.len(),
-                            position: 0,
+                Field::Text(TextField::DomainNoTokenizer) => {
+                    let domain = cache.pretokenize_domain();
+
+                    doc.add_pre_tokenized_text(
+                        tantivy_field,
+                        PreTokenizedString {
                             text: domain.text.clone(),
-                            position_length: 1,
-                        }],
-                    },
-                ),
+                            tokens: vec![tantivy::tokenizer::Token {
+                                offset_from: 0,
+                                offset_to: domain.text.len(),
+                                position: 0,
+                                text: domain.text.clone(),
+                                position_length: 1,
+                            }],
+                        },
+                    )
+                }
                 Field::Text(TextField::TitleIfHomepage) => {
+                    let title = cache
+                        .pretokenize_title()
+                        .as_ref()
+                        .map(Clone::clone)
+                        .map_err(|e| anyhow::anyhow!("{}", e))?;
+
                     if self.is_homepage() {
-                        doc.add_pre_tokenized_text(tantivy_field, title.clone());
+                        doc.add_pre_tokenized_text(tantivy_field, title);
                     } else {
                         doc.add_text(tantivy_field, "");
                     }
                 }
                 Field::Text(TextField::DomainIfHomepage) => {
+                    let domain = cache.pretokenize_domain();
                     if self.is_homepage() {
                         doc.add_text(tantivy_field, domain.text.clone());
                     } else {
@@ -359,31 +379,35 @@ impl Html {
                     }
                 }
                 Field::Text(TextField::DomainNameNoTokenizer) => {
+                    let domain_name = cache.domain_name();
+
                     doc.add_pre_tokenized_text(
                         tantivy_field,
                         PreTokenizedString {
-                            text: domain_name.to_string(),
+                            text: domain_name.clone(),
                             tokens: vec![tantivy::tokenizer::Token {
                                 offset_from: 0,
                                 offset_to: domain_name.len(),
                                 position: 0,
-                                text: domain_name.to_string(),
+                                text: domain_name.clone(),
                                 position_length: 1,
                             }],
                         },
                     );
                 }
                 Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
+                    let domain_name = cache.domain_name();
+
                     if self.is_homepage() {
                         doc.add_pre_tokenized_text(
                             tantivy_field,
                             PreTokenizedString {
-                                text: domain_name.to_string(),
+                                text: domain_name.clone(),
                                 tokens: vec![tantivy::tokenizer::Token {
                                     offset_from: 0,
                                     offset_to: domain_name.len(),
                                     position: 0,
-                                    text: domain_name.to_string(),
+                                    text: domain_name.clone(),
                                     position_length: 1,
                                 }],
                             },
@@ -393,7 +417,9 @@ impl Html {
                     }
                 }
                 Field::Text(TextField::DomainIfHomepageNoTokenizer) => {
-                    if self.is_homepage() {
+                    let domain = cache.pretokenize_domain();
+
+                    if self.url().is_homepage() {
                         doc.add_pre_tokenized_text(
                             tantivy_field,
                             PreTokenizedString {
@@ -412,19 +438,34 @@ impl Html {
                     }
                 }
                 Field::Text(TextField::AllBody) => {
+                    let all_text = cache
+                        .pretokenize_all_text()
+                        .as_ref()
+                        .map(Clone::clone)
+                        .map_err(|e| anyhow::anyhow!("{}", e))?;
+
                     doc.add_pre_tokenized_text(tantivy_field, all_text.clone())
                 }
                 Field::Text(TextField::RecipeFirstIngredientTagId) => {
-                    doc.add_text(tantivy_field, first_ingredient_tag_id.clone());
+                    doc.add_text(
+                        tantivy_field,
+                        cache.first_ingredient_tag_id().cloned().unwrap_or_default(),
+                    );
                 }
                 Field::Text(TextField::SchemaOrgJson) => {
-                    doc.add_text(tantivy_field, schema_json.clone());
+                    doc.add_text(tantivy_field, cache.schema_json());
                 }
                 Field::Text(TextField::FlattenedSchemaOrgJson) => {
-                    doc.add_pre_tokenized_text(tantivy_field, pretokenized_schema_json.clone());
+                    doc.add_pre_tokenized_text(
+                        tantivy_field,
+                        cache.pretokenized_schema_json().clone(),
+                    );
                 }
                 Field::Text(TextField::MicroformatTags) => {
-                    doc.add_pre_tokenized_text(tantivy_field, microformats.clone());
+                    doc.add_pre_tokenized_text(
+                        tantivy_field,
+                        cache.pretokenize_microformats().clone(),
+                    );
                 }
                 Field::Fast(FastField::IsHomepage) => {
                     doc.add_u64(tantivy_field, (self.is_homepage()).into());
@@ -438,66 +479,79 @@ impl Html {
                     doc.add_u64(tantivy_field, self.trackers().len() as u64)
                 }
                 Field::Fast(FastField::NumUrlTokens) => {
-                    doc.add_u64(tantivy_field, url.tokens.len() as u64)
-                }
-                Field::Fast(FastField::NumMicroformatTagsTokens) => {
-                    doc.add_u64(tantivy_field, microformats.tokens.len() as u64)
-                }
-                Field::Fast(FastField::NumTitleTokens) => {
-                    doc.add_u64(tantivy_field, title.tokens.len() as u64)
-                }
-                Field::Fast(FastField::NumCleanBodyTokens) => {
-                    doc.add_u64(tantivy_field, clean_text.tokens.len() as u64)
-                }
-                Field::Fast(FastField::NumDescriptionTokens) => {
-                    doc.add_u64(tantivy_field, description.tokens.len() as u64)
-                }
-                Field::Fast(FastField::NumUrlForSiteOperatorTokens) => {
-                    doc.add_u64(tantivy_field, url_for_site_operator.tokens.len() as u64)
-                }
-                Field::Fast(FastField::NumDomainTokens) => {
-                    doc.add_u64(tantivy_field, domain.tokens.len() as u64)
-                }
-                Field::Fast(FastField::NumFlattenedSchemaTokens) => {
-                    doc.add_u64(tantivy_field, pretokenized_schema_json.tokens.len() as u64)
+                    doc.add_u64(tantivy_field, cache.pretokenize_url().tokens.len() as u64)
                 }
+                Field::Fast(FastField::NumMicroformatTagsTokens) => doc.add_u64(
+                    tantivy_field,
+                    cache.pretokenize_microformats().tokens.len() as u64,
+                ),
+                Field::Fast(FastField::NumTitleTokens) => doc.add_u64(
+                    tantivy_field,
+                    cache
+                        .pretokenize_title()
+                        .as_ref()
+                        .map(|n| n.tokens.len() as u64)
+                        .map_err(|e| anyhow::anyhow!("{}", e))?,
+                ),
+                Field::Fast(FastField::NumCleanBodyTokens) => doc.add_u64(
+                    tantivy_field,
+                    cache.pretokenize_clean_text().tokens.len() as u64,
+                ),
+                Field::Fast(FastField::NumDescriptionTokens) => doc.add_u64(
+                    tantivy_field,
+                    cache.pretokenize_description().tokens.len() as u64,
+                ),
+                Field::Fast(FastField::NumUrlForSiteOperatorTokens) => doc.add_u64(
+                    tantivy_field,
+                    cache.pretokenize_url_for_site_operator().tokens.len() as u64,
+                ),
+                Field::Fast(FastField::NumDomainTokens) => doc.add_u64(
+                    tantivy_field,
+                    cache.pretokenize_domain().tokens.len() as u64,
+                ),
+                Field::Fast(FastField::NumFlattenedSchemaTokens) => doc.add_u64(
+                    tantivy_field,
+                    cache.pretokenized_schema_json().tokens.len() as u64,
+                ),
                 Field::Fast(FastField::SiteHash1) => {
-                    doc.add_u64(tantivy_field, site_hash[0]);
+                    doc.add_u64(tantivy_field, cache.site_hash()[0]);
                 }
                 Field::Fast(FastField::SiteHash2) => {
-                    doc.add_u64(tantivy_field, site_hash[1]);
+                    doc.add_u64(tantivy_field, cache.site_hash()[1]);
                 }
                 Field::Fast(FastField::UrlWithoutQueryHash1) => {
-                    doc.add_u64(tantivy_field, url_without_query_hash[0]);
+                    doc.add_u64(tantivy_field, cache.url_without_query_hash()[0]);
                 }
                 Field::Fast(FastField::UrlWithoutQueryHash2) => {
-                    doc.add_u64(tantivy_field, url_without_query_hash[1]);
+                    doc.add_u64(tantivy_field, cache.url_without_query_hash()[1]);
                 }
                 Field::Fast(FastField::UrlHash1) => {
-                    doc.add_u64(tantivy_field, url_hash[0]);
+                    doc.add_u64(tantivy_field, cache.url_hash()[0]);
                 }
                 Field::Fast(FastField::UrlHash2) => {
-                    doc.add_u64(tantivy_field, url_hash[1]);
+                    doc.add_u64(tantivy_field, cache.url_hash()[1]);
                 }
                 Field::Fast(FastField::UrlWithoutTldHash1) => {
-                    doc.add_u64(tantivy_field, url_without_tld_hash[0]);
+                    doc.add_u64(tantivy_field, cache.url_without_tld_hash()[0]);
                 }
                 Field::Fast(FastField::UrlWithoutTldHash2) => {
-                    doc.add_u64(tantivy_field, url_without_tld_hash[1]);
+                    doc.add_u64(tantivy_field, cache.url_without_tld_hash()[1]);
                 }
                 Field::Fast(FastField::DomainHash1) => {
-                    doc.add_u64(tantivy_field, domain_hash[0]);
+                    doc.add_u64(tantivy_field, cache.domain_hash()[0]);
                 }
                 Field::Fast(FastField::DomainHash2) => {
-                    doc.add_u64(tantivy_field, domain_hash[1]);
+                    doc.add_u64(tantivy_field, cache.domain_hash()[1]);
                 }
                 Field::Fast(FastField::TitleHash1) => {
-                    doc.add_u64(tantivy_field, title_hash[0]);
+                    doc.add_u64(tantivy_field, cache.title_hash()[0]);
                 }
                 Field::Fast(FastField::TitleHash2) => {
-                    doc.add_u64(tantivy_field, title_hash[1]);
+                    doc.add_u64(tantivy_field, cache.title_hash()[1]);
                 }
                 Field::Fast(FastField::SimHash) => {
+                    let clean_text = cache.pretokenize_clean_text();
+
                     let hash = if !clean_text.text.is_empty() {
                         simhash::hash(&clean_text.text)
                     } else {
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index 0b93b31e..153fe5c8 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -28,6 +28,7 @@ use super::{adservers::AD_SERVERS, schema_org, Meta, Script};
 
 use super::url_ext::UrlExt;
 
+mod fn_cache;
 mod into_tantivy;
 mod links;
 mod microformats;