extract keywords using rake algorithm

crates/core/src/lib.rs

@@ -61,6 +61,7 @@ mod models;
 pub mod naive_bayes;
 pub mod prehashed;
 mod query;
+mod rake;
 pub mod ranking;
 mod schema;
 mod search_ctx;
@@ -69,6 +70,7 @@ pub mod searcher;
 mod simhash;
 pub mod similar_hosts;
 mod snippet;
+mod stopwords;
 pub mod summarizer;
 mod tokenizer;
 #[allow(unused)]

crates/core/src/rake/mod.rs

@@ -0,0 +1,273 @@
+//! Implementation of the RAKE keyword extraction algorithm.
+//!
+//! https://doi.org/10.1002/9780470689646.ch1
+//!
+//! We have modified the algorithm a bit so that it generates
+//! the keywords based on a summary of the text as extracted
+//! by the sentences with most frequent words.
+
+use hashbrown::{HashMap, HashSet};
+use itertools::Itertools;
+use whatlang::Lang;
+
+use crate::stopwords;
+
+fn is_sent_split(c: char) -> bool {
+    matches!(
+        c,
+        '.' | '!' | '?' | '\n' | '\r' | '\t' | '\u{2026}' | '\u{2025}' | '\u{2024}'
+    )
+}
+
+fn sentences(text: &str) -> impl Iterator<Item = Sentence<'_>> {
+    text.split_terminator(is_sent_split).map(Sentence)
+}
+
+fn phrases<'a>(
+    sentence: &'a Sentence<'a>,
+    stopwords: &'a HashSet<String>,
+    max_words: usize,
+) -> Vec<Phrase> {
+    sentence
+        .0
+        .split_whitespace()
+        .map(|word| word.replace(|c: char| matches!(c, ',' | '.'), ""))
+        .group_by(|word| !stopwords.contains(word))
+        .into_iter()
+        .filter_map(move |(is_keyword, words)| {
+            if is_keyword {
+                let mut phrase = Vec::new();
+                let mut num_words = 0;
+
+                for word in words {
+                    phrase.push(word);
+                    num_words += 1;
+                }
+
+                if num_words > 1 && num_words <= max_words {
+                    return Some(phrase);
+                }
+            }
+
+            None
+        })
+        .map(Phrase)
+        .collect()
+}
+
+fn smmry<'a, 'b>(
+    sents: Vec<Sentence<'a>>,
+    stopwords: &'b HashSet<String>,
+    top_sentences: usize,
+) -> Vec<Sentence<'a>> {
+    let mut word_frequency = HashMap::new();
+
+    for sentence in &sents {
+        for word in sentence.0.split_whitespace() {
+            if !stopwords.contains(word) {
+                *word_frequency.entry(word.to_string()).or_insert(0) += 1;
+            }
+        }
+    }
+
+    let scored_sentences = sents
+        .into_iter()
+        .filter_map(|sentence| {
+            let words = sentence.0.split_whitespace().collect::<Vec<_>>();
+
+            if words.is_empty() {
+                return None;
+            }
+
+            let score = words
+                .iter()
+                .map(|word| word_frequency.get(*word).unwrap_or(&0))
+                .sum::<u64>();
+            Some(ScoredSentence { sentence, score })
+        })
+        .collect::<Vec<_>>();
+
+    scored_sentences
+        .into_iter()
+        .sorted_by(|a, b| b.score.cmp(&a.score))
+        .take(top_sentences)
+        .map(|s| s.sentence)
+        .collect::<Vec<_>>()
+}
+
+/// A sentence is a sequence of words from the input text
+/// that has been split by punctuation.
+#[derive(Debug)]
+struct Sentence<'a>(&'a str);
+
+#[derive(Debug)]
+struct ScoredSentence<'a> {
+    sentence: Sentence<'a>,
+    score: u64,
+}
+
+type Word = String;
+/// A phrase is a sub-sequence of a sentence that has been
+/// split by stopwords.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct Phrase(Vec<Word>);
+
+#[derive(Debug, Clone)]
+pub struct Keyword {
+    pub keyword: String,
+    pub score: f64,
+}
+
+pub struct RakeParams {
+    pub summary_sentences: usize,
+    pub max_words: usize,
+}
+
+impl Default for RakeParams {
+    fn default() -> Self {
+        Self {
+            summary_sentences: 16,
+            max_words: 5,
+        }
+    }
+}
+
+pub struct RakeModel {
+    stopwords: HashMap<Lang, HashSet<String>>,
+    params: RakeParams,
+}
+
+impl Default for RakeModel {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl RakeModel {
+    pub fn new() -> Self {
+        Self::with_params(RakeParams::default())
+    }
+
+    pub fn with_params(params: RakeParams) -> Self {
+        let stopwords = stopwords::all().clone();
+        Self { stopwords, params }
+    }
+
+    pub fn keywords(&self, text: &str, lang: Lang) -> Vec<Keyword> {
+        let text = text.to_lowercase();
+        let stopwords = self
+            .stopwords
+            .get(&lang)
+            .unwrap_or_else(|| self.stopwords.get(&Lang::Eng).expect("English stopwords"));
+
+        let sentences = sentences(&text).collect::<Vec<_>>();
+        let top_sentences = smmry(sentences, stopwords, self.params.summary_sentences);
+
+        let phrases = top_sentences
+            .into_iter()
+            .flat_map(|sentence| phrases(&sentence, stopwords, self.params.max_words))
+            .collect::<Vec<_>>();
+
+        let num_words = phrases.iter().map(|p| p.0.len()).sum::<usize>();
+
+        let mut word_frequency = HashMap::with_capacity(num_words);
+        let mut word_degree = HashMap::with_capacity(num_words);
+
+        for phrase in &phrases {
+            let words = &phrase.0;
+            let degree = words.len() as f64 - 1.0;
+            for word in words {
+                *word_frequency.entry(word.to_string()).or_insert(0.0) += 1.0;
+                *word_degree.entry(word.to_string()).or_insert(0.0) += degree;
+            }
+        }
+
+        let mut keywords = HashMap::with_capacity(num_words);
+        for phrase in phrases {
+            let words = &phrase.0;
+            let mut score = 0.0;
+            for word in words {
+                score += word_degree[word] / word_frequency[word];
+            }
+            score /= words.len() as f64;
+            keywords.insert(phrase, score);
+        }
+
+        keywords
+            .into_iter()
+            .sorted_by(|(_, score1), (_, score2)| score2.partial_cmp(score1).unwrap())
+            .map(|(phrase, score)| Keyword {
+                keyword: phrase.0.join(" "),
+                score,
+            })
+            .take(word_degree.len() / 3)
+            .filter(|k| k.keyword.len() > 1)
+            .filter(|k| k.score > 0.0)
+            .collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const TURING_TEXT: &str = r##"
+    Alan Mathison Turing OBE FRS (/ˈtjʊərɪŋ/; 23 June 1912 – 7 June 1954) was an English mathematician, computer scientist, logician, cryptanalyst, philosopher and theoretical biologist.[5] Turing was highly influential in the development of theoretical computer science, providing a formalisation of the concepts of algorithm and computation with the Turing machine, which can be considered a model of a general-purpose computer.[6][7][8] He is widely considered to be the father of theoretical computer science and artificial intelligence.[9]
+
+Born in Maida Vale, London, Turing was raised in southern England. He graduated from King's College, Cambridge, with a degree in mathematics. Whilst he was a fellow at Cambridge, he published a proof demonstrating that some purely mathematical yes–no questions can never be answered by computation. He defined a Turing machine and proved that the halting problem for Turing machines is undecidable. In 1938, he earned his PhD from the Department of Mathematics at Princeton University.
+
+During the Second World War, Turing worked for the Government Code and Cypher School at Bletchley Park, Britain's codebreaking centre that produced Ultra intelligence. For a time he led Hut 8, the section that was responsible for German naval cryptanalysis. Here, he devised a number of techniques for speeding the breaking of German ciphers, including improvements to the pre-war Polish bomba method, an electromechanical machine that could find settings for the Enigma machine. Turing played a crucial role in cracking intercepted coded messages that enabled the Allies to defeat the Axis powers in many crucial engagements, including the Battle of the Atlantic.[10][11]
+
+After the war, Turing worked at the National Physical Laboratory, where he designed the Automatic Computing Engine, one of the first designs for a stored-program computer. In 1948, Turing joined Max Newman's Computing Machine Laboratory at the Victoria University of Manchester, where he helped develop the Manchester computers[12] and became interested in mathematical biology. He wrote a paper on the chemical basis of morphogenesis[13][1] and predicted oscillating chemical reactions such as the Belousov–Zhabotinsky reaction, first observed in the 1960s. Despite these accomplishments, Turing was never fully recognised in Britain during his lifetime because much of his work was covered by the Official Secrets Act.[14]
+
+Turing was prosecuted in 1952 for homosexual acts. He accepted hormone treatment with DES, a procedure commonly referred to as chemical castration, as an alternative to prison. Turing died on 7 June 1954, 16 days before his 42nd birthday, from cyanide poisoning. An inquest determined his death as a suicide, but it has been noted that the known evidence is also consistent with accidental poisoning. Following a public campaign in 2009, British prime minister Gordon Brown made an official public apology on behalf of the government for "the appalling way [Turing] was treated". Queen Elizabeth II granted a posthumous pardon in 2013. The term "Alan Turing law" is now used informally to refer to a 2017 law in the United Kingdom that retroactively pardoned men cautioned or convicted under historical legislation that outlawed homosexual acts.[15]
+
+Turing has an extensive legacy with statues of him and many things named after him, including an annual award for computer science innovations. He appears on the current Bank of England £50 note, which was released on 23 June 2021 to coincide with his birthday. A 2019 BBC series, as voted by the audience, named him the greatest person of the 20th century.
+Early life and education
+Family
+English Heritage plaque in Maida Vale, London marking Turing's birthplace in 1912
+
+Turing was born in Maida Vale, London, while his father, Julius Mathison Turing was on leave from his position with the Indian Civil Service (ICS) of the British Raj government at Chatrapur, then in the Madras Presidency and presently in Odisha state, in India.[16][17] Turing's father was the son of a clergyman, the Rev. John Robert Turing, from a Scottish family of merchants that had been based in the Netherlands and included a baronet. Turing's mother, Julius's wife, was Ethel Sara Turing (née Stoney), daughter of Edward Waller Stoney, chief engineer of the Madras Railways. The Stoneys were a Protestant Anglo-Irish gentry family from both County Tipperary and County Longford, while Ethel herself had spent much of her childhood in County Clare.[18] Julius and Ethel married on 1 October 1907 at Bartholomew's church on Clyde Road, in Dublin.[19]
+
+Julius's work with the ICS brought the family to British India, where his grandfather had been a general in the Bengal Army. However, both Julius and Ethel wanted their children to be brought up in Britain, so they moved to Maida Vale,[20] London, where Alan Turing was born on 23 June 1912, as recorded by a blue plaque on the outside of the house of his birth,[21][22] later the Colonnade Hotel.[16][23] Turing had an elder brother, John Ferrier Turing, father of Sir John Dermot Turing, 12th Baronet of the Turing baronets.[24]
+
+Turing's father's civil service commission was still active during Turing's childhood years, and his parents travelled between Hastings in the United Kingdom[25] and India, leaving their two sons to stay with a retired Army couple. At Hastings, Turing stayed at Baston Lodge, Upper Maze Hill, St Leonards-on-Sea, now marked with a blue plaque.[26] The plaque was unveiled on 23 June 2012, the centenary of Turing's birth.[27]
+
+Very early in life, Turing showed signs of the genius that he was later to display prominently.[28] His parents purchased a house in Guildford in 1927, and Turing lived there during school holidays. The location is also marked with a blue plaque.[29]
+School
+
+Turing's parents enrolled him at St Michael's, a primary school at 20 Charles Road, St Leonards-on-Sea, from the age of six to nine. The headmistress recognised his talent, noting that she has "...had clever boys and hardworking boys, but Alan is a genius".[30]
+
+Between January 1922 and 1926, Turing was educated at Hazelhurst Preparatory School, an independent school in the village of Frant in Sussex (now East Sussex).[31] In 1926, at the age of 13, he went on to Sherborne School,[32] an independent boarding school in the market town of Sherborne in Dorset, where he boarded at Westcott House. The first day of term coincided with the 1926 General Strike, in Britain, but Turing was so determined to attend that he rode his bicycle unaccompanied 60 miles (97 km) from Southampton to Sherborne, stopping overnight at an inn.[33]
+
+Turing's natural inclination towards mathematics and science did not earn him respect from some of the teachers at Sherborne, whose definition of education placed more emphasis on the classics. His headmaster wrote to his parents: "I hope he will not fall between two stools. If he is to stay at public school, he must aim at becoming educated. If he is to be solely a Scientific Specialist, he is wasting his time at a public school".[34] Despite this, Turing continued to show remarkable ability in the studies he loved, solving advanced problems in 1927 without having studied even elementary calculus. In 1928, aged 16, Turing encountered Albert Einstein's work; not only did he grasp it, but it is possible that he managed to deduce Einstein's questioning of Newton's laws of motion from a text in which this was never made explicit.[35]
+Christopher Morcom
+
+At Sherborne, Turing formed a significant friendship with fellow pupil Christopher Collan Morcom (13 July 1911 – 13 February 1930),[36] who has been described as Turing's first love.[37][38][39] Their relationship provided inspiration in Turing's future endeavours, but it was cut short by Morcom's death, in February 1930, from complications of bovine tuberculosis, contracted after drinking infected cow's milk some years previously.[40][41][42]
+
+The event caused Turing great sorrow. He coped with his grief by working that much harder on the topics of science and mathematics that he had shared with Morcom. In a letter to Morcom's mother, Frances Isobel Morcom (née Swan), Turing wrote:
+
+    I am sure I could not have found anywhere another companion so brilliant and yet so charming and unconceited. I regarded my interest in my work, and in such things as astronomy (to which he introduced me) as something to be shared with him and I think he felt a little the same about me ... I know I must put as much energy if not as much interest into my work as if he were alive, because that is what he would like me to do.[43]
+
+Turing's relationship with Morcom's mother continued long after Morcom's death, with her sending gifts to Turing, and him sending letters, typically on Morcom's birthday.[44] A day before the third anniversary of Morcom's death (13 February 1933), he wrote to Mrs. Morcom:
+
+    I expect you will be thinking of Chris when this reaches you. I shall too, and this letter is just to tell you that I shall be thinking of Chris and of you tomorrow. I am sure that he is as happy now as he was when he was here. Your affectionate Alan.[45]
+
+Some have speculated that Morcom's death was the cause of Turing's atheism and materialism.[46] Apparently, at this point in his life he still believed in such concepts as a spirit, independent of the body and surviving death. In a later letter, also written to Morcom's mother, Turing wrote:
+
+    Personally, I believe that spirit is really eternally connected with matter but certainly not by the same kind of body ... as regards the actual connection between spirit and body I consider that the body can hold on to a 'spirit', whilst the body is alive and awake the two are firmly connected. When the body is asleep I cannot guess what happens but when the body dies, the 'mechanism' of the body, holding the spirit is gone and the spirit finds a new body sooner or later, perhaps immediately.[47][48]
+
+University and work on computability
+
+After graduating from Sherborne, Turing applied for several Cambridge colleges scholarships, including Trinity and King's, eventually earning a £80 per annum scholarship (equivalent to about £4,300 as of 2023) to study at the latter.[49][50] There, Turing studied the undergraduate course in Schedule B (that is, a three-year Parts I and II, of the Mathematical Tripos, with extra courses at the end of the third year, as Part III only emerged as a separate degree in 1934) from February 1931 to November 1934 at King's College, Cambridge, where he was awarded first-class honours in mathematics. His dissertation, On the Gaussian error function, written during his senior year and delivered in November 1934 (with a deadline date of 6 December) proved a version of the central limit theorem. It was finally accepted on 16 March 1935. By spring of that same year, Turing started his master's course (Part III)—which he completed in 1937—and, at the same time, he published his first paper, a one-page article called Equivalence of left and right almost periodicity (sent on 23 April), featured in the tenth volume of the Journal of the London Mathematical Society.[51] Later that year, Turing was elected a Fellow of King's College on the strength of his dissertation.[52] However, and, unknown to Turing, this version of the theorem he proved in his paper, had already been proven, in 1922, by Jarl Waldemar Lindeberg. Despite this, the committee found Turing's methods original and so regarded the work worthy of consideration for the fellowship. Abram Besicovitch's report for the committee went so far as to say that if Turing's work had been published before Lindeberg's, it would have been "an important event in the mathematical literature of that year".[53][54][55]
+
+Between the springs of 1935 and 1936, at the same time as Church, Turing worked on the decidability of problems, starting from Godel's incompleteness theorems. In mid-April 1936, Turing sent Max Newman the first draft typescript of his investigations. That same month, Alonzo Church published his An Unsolvable Problem of Elementary Number Theory, with similar conclusions to Turing's then-yet unpublished work. Finally, on 28 May of that year, he finished and delivered his 36-page paper for publication called "On Computable Numbers, with an Application to the Entscheidungsproblem".[56] It was published in the Proceedings of the London Mathematical Society journal in two parts, the first on 30 November and the second on 23 December.[57] In this paper, Turing reformulated Kurt Gödel's 1931 results on the limits of proof and computation, replacing Gödel's universal arithmetic-based formal language with the formal and simple hypothetical devices that became known as Turing machines. The Entscheidungsproblem (decision problem) was originally posed by German mathematician David Hilbert in 1928. Turing proved that his "universal computing machine" would be capable of performing any conceivable mathematical computation if it were representable as an algorithm. He went on to prove that there was no solution to the decision problem by first showing that the halting problem for Turing machines is undecidable: it is not possible to decide algorithmically whether a Turing machine will ever halt. This paper has been called "easily the most influential math paper in history".[58] "##;
+
+    #[test]
+    fn test_keywords() {
+        let rake = RakeModel::new();
+        let keywords = rake.keywords(TURING_TEXT, Lang::Eng);
+
+        assert!(!keywords.is_empty());
+    }
+}