From 5ce97abf46ffbd9e7fb6dc89efb3d5360f2a74df Mon Sep 17 00:00:00 2001 From: Mikkel Denker Date: Wed, 13 Mar 2024 09:48:07 +0100 Subject: [PATCH] Run frontend lint in CI (#180) Adds `npm run lint` to CI and fixes all the previous lint errors. --- crates/core/src/api/autosuggest.rs | 53 +++++++++++++----- crates/core/src/api/docs.rs | 4 +- crates/core/src/highlighted.rs | 55 +++++++++++++++++++ crates/core/src/inverted_index.rs | 5 +- crates/core/src/lib.rs | 1 + crates/core/src/search_prettifier/mod.rs | 24 +++++--- crates/core/src/snippet.rs | 54 +++++------------- frontend/.eslintignore | 3 + frontend/.eslintrc.cjs | 6 ++ frontend/package.json | 2 +- frontend/src/lib/api/index.ts | 21 +++---- frontend/src/lib/components/Searchbar.svelte | 28 +++++----- frontend/src/lib/components/Select.svelte | 6 +- frontend/src/lib/rankings.ts | 2 +- frontend/src/routes/about/+page.svelte | 1 + .../privacy-and-happy-lawyers/+page.svelte | 1 + frontend/src/routes/search/+page.svelte | 8 ++- .../src/routes/search/DirectAnswer.svelte | 26 --------- frontend/src/routes/search/Entity.svelte | 2 +- frontend/src/routes/webmasters/+page.svelte | 1 + frontend/tsconfig.json | 3 +- scripts/ci/check | 4 +- 22 files changed, 183 insertions(+), 127 deletions(-) create mode 100644 crates/core/src/highlighted.rs delete mode 100644 frontend/src/routes/search/DirectAnswer.svelte diff --git a/crates/core/src/api/autosuggest.rs b/crates/core/src/api/autosuggest.rs index 9a1d973e..d6cfe6ce 100644 --- a/crates/core/src/api/autosuggest.rs +++ b/crates/core/src/api/autosuggest.rs @@ -20,29 +20,32 @@ use axum::{extract, response::IntoResponse, Json}; use serde::Serialize; use utoipa::{IntoParams, ToSchema}; -use super::State; +use crate::highlighted::HighlightedFragment; -const HIGHLIGHTED_PREFIX: &str = ""; -const HIGHLIGHTED_POSTFIX: &str = ""; +use super::State; -fn highlight(query: &str, suggestion: &str) -> String { +fn highlight(query: &str, suggestion: &str) -> Vec { let idx = suggestion .chars() .zip(query.chars()) .position(|(suggestion_char, query_char)| suggestion_char != query_char) .unwrap_or(query.chars().count()); - let mut new_suggestion: String = suggestion.chars().take(idx).collect(); - new_suggestion += HIGHLIGHTED_PREFIX; - new_suggestion += suggestion.chars().skip(idx).collect::().as_str(); - new_suggestion += HIGHLIGHTED_POSTFIX; + let mut new_suggestion = vec![HighlightedFragment::new_unhighlighted( + suggestion.chars().take(idx).collect(), + )]; + + new_suggestion.push(HighlightedFragment::new_highlighted( + suggestion.chars().skip(idx).collect::(), + )); + new_suggestion } #[derive(Serialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct Suggestion { - highlighted: String, + highlighted: Vec, raw: String, } @@ -95,28 +98,50 @@ pub async fn browser( #[cfg(test)] mod tests { + use crate::highlighted::HighlightedKind; + use super::*; + const HIGHLIGHTED_PREFIX: &str = ""; + const HIGHLIGHTED_POSTFIX: &str = ""; + + fn highlight_fragments(frags: &[HighlightedFragment]) -> String { + frags + .iter() + .map(|frag| match frag.kind { + HighlightedKind::Normal => frag.text.clone(), + HighlightedKind::Highlighted => { + format!( + "{}{}{}", + HIGHLIGHTED_PREFIX, + frag.text(), + HIGHLIGHTED_POSTFIX + ) + } + }) + .collect() + } + #[test] fn suffix_highlight() { assert_eq!( - highlight("", "test"), + highlight_fragments(&highlight("", "test")), format!("{HIGHLIGHTED_PREFIX}test{HIGHLIGHTED_POSTFIX}") ); assert_eq!( - highlight("t", "test"), + highlight_fragments(&highlight("t", "test")), format!("t{HIGHLIGHTED_PREFIX}est{HIGHLIGHTED_POSTFIX}") ); assert_eq!( - highlight("te", "test"), + highlight_fragments(&highlight("te", "test")), format!("te{HIGHLIGHTED_PREFIX}st{HIGHLIGHTED_POSTFIX}") ); assert_eq!( - highlight("tes", "test"), + highlight_fragments(&highlight("tes", "test")), format!("tes{HIGHLIGHTED_PREFIX}t{HIGHLIGHTED_POSTFIX}") ); assert_eq!( - highlight("test", "test"), + highlight_fragments(&highlight("test", "test")), format!("test{HIGHLIGHTED_PREFIX}{HIGHLIGHTED_POSTFIX}") ); } diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rs index 6cd02e29..28248c31 100644 --- a/crates/core/src/api/docs.rs +++ b/crates/core/src/api/docs.rs @@ -59,8 +59,8 @@ use utoipa_swagger_ui::SwaggerUi; crate::search_prettifier::CodeOrText, crate::snippet::TextSnippet, - crate::snippet::TextSnippetFragment, - crate::snippet::TextSnippetFragmentKind, + crate::highlighted::HighlightedFragment, + crate::highlighted::HighlightedKind, crate::entity_index::entity::EntitySnippet, crate::entity_index::entity::EntitySnippetFragment, diff --git a/crates/core/src/highlighted.rs b/crates/core/src/highlighted.rs new file mode 100644 index 00000000..2191b4a4 --- /dev/null +++ b/crates/core/src/highlighted.rs @@ -0,0 +1,55 @@ +// Stract is an open source web search engine. +// Copyright (C) 2024 Stract ApS +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use utoipa::ToSchema; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +pub enum HighlightedKind { + Normal, + Highlighted, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct HighlightedFragment { + pub kind: HighlightedKind, + pub text: String, +} + +impl HighlightedFragment { + pub fn new_unhighlighted(text: String) -> Self { + Self::new_normal(text) + } + + pub fn new_normal(text: String) -> Self { + Self { + kind: HighlightedKind::Normal, + text, + } + } + + pub fn new_highlighted(text: String) -> Self { + Self { + kind: HighlightedKind::Highlighted, + text, + } + } + + pub fn text(&self) -> &str { + &self.text + } +} diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rs index c1901c36..e026718f 100644 --- a/crates/core/src/inverted_index.rs +++ b/crates/core/src/inverted_index.rs @@ -39,6 +39,7 @@ use url::Url; use crate::collector::{Hashes, MainCollector}; use crate::config::SnippetConfig; use crate::fastfield_reader::FastFieldReader; +use crate::highlighted::HighlightedFragment; use crate::query::shortcircuit::ShortCircuitQuery; use crate::query::Query; use crate::ranking::initial::Score; @@ -46,8 +47,8 @@ use crate::ranking::pipeline::RecallRankingWebpage; use crate::ranking::SignalAggregator; use crate::schema::{FastField, Field, TextField}; use crate::search_ctx::Ctx; +use crate::snippet; use crate::snippet::TextSnippet; -use crate::snippet::{self, TextSnippetFragment}; use crate::tokenizer::{ BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, TrigramTokenizer, }; @@ -475,7 +476,7 @@ impl InvertedIndex { }; page.snippet = TextSnippet { - fragments: vec![TextSnippetFragment::new_unhighlighted(snippet)], + fragments: vec![HighlightedFragment::new_unhighlighted(snippet)], }; } else { let min_body_len = if url.is_homepage() { diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index caf64a94..e3d9f0c4 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -51,6 +51,7 @@ mod executor; mod external_sort; mod fastfield_reader; pub mod feed; +mod highlighted; mod human_website_annotations; pub mod hyperloglog; pub mod image_store; diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rs index 64fd9793..dbaa49fa 100644 --- a/crates/core/src/search_prettifier/mod.rs +++ b/crates/core/src/search_prettifier/mod.rs @@ -25,6 +25,7 @@ use url::Url; use utoipa::ToSchema; use crate::{ + highlighted::HighlightedFragment, inverted_index::RetrievedWebpage, ranking::{Signal, SignalScore}, snippet::TextSnippet, @@ -57,12 +58,12 @@ pub enum RichSnippet { #[serde(rename_all = "camelCase")] pub struct HighlightedSpellCorrection { pub raw: String, - pub highlighted: String, + pub highlighted: Vec, } impl From for HighlightedSpellCorrection { fn from(correction: web_spell::Correction) -> Self { - let mut highlighted = String::new(); + let mut highlighted = Vec::new(); let mut raw = String::new(); for term in correction.terms { @@ -71,22 +72,27 @@ impl From for HighlightedSpellCorrection { orig: _, correction, } => { - highlighted - .push_str(&("".to_string() + correction.as_str() + "")); + let mut correction = correction.trim().to_string(); + correction.push(' '); + + highlighted.push(HighlightedFragment::new_highlighted(correction.clone())); raw.push_str(&correction); } CorrectionTerm::NotCorrected(orig) => { - highlighted.push_str(&orig); + let mut orig = orig.trim().to_string(); + orig.push(' '); + + highlighted.push(HighlightedFragment::new_normal(orig.clone())); raw.push_str(&orig); } } - - raw.push(' '); - highlighted.push(' '); } raw = raw.trim_end().to_string(); - highlighted = highlighted.trim_end().to_string(); + + if let Some(last) = highlighted.last_mut() { + last.text = last.text.trim_end().to_string(); + } Self { raw, highlighted } } diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs index 1875d8f7..df1b4e44 100644 --- a/crates/core/src/snippet.rs +++ b/crates/core/src/snippet.rs @@ -17,6 +17,7 @@ use std::ops::Range; use crate::config::SnippetConfig; +use crate::highlighted::{HighlightedFragment, HighlightedKind}; use crate::query::Query; use crate::tokenizer::{BigramTokenizer, Normal, Stemmed, Tokenizer, TrigramTokenizer}; use crate::web_spell::sentence_ranges; @@ -46,37 +47,10 @@ struct PassageCandidate { doc_terms: HashMap, } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)] -#[serde(rename_all = "camelCase")] -pub enum TextSnippetFragmentKind { - Normal, - Highlighted, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)] -#[serde(rename_all = "camelCase")] -pub struct TextSnippetFragment { - pub kind: TextSnippetFragmentKind, - text: String, -} - -impl TextSnippetFragment { - pub fn new_unhighlighted(text: String) -> Self { - TextSnippetFragment { - kind: TextSnippetFragmentKind::Normal, - text, - } - } - - pub fn text(&self) -> &str { - &self.text - } -} - #[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] pub struct TextSnippet { - pub fragments: Vec, + pub fragments: Vec, } impl TextSnippet { @@ -125,14 +99,14 @@ impl SnippetBuilder { for range in self.highlights { if range.start > last_end { - fragments.push(TextSnippetFragment { - kind: TextSnippetFragmentKind::Normal, + fragments.push(HighlightedFragment { + kind: HighlightedKind::Normal, text: self.fragment[last_end..range.start].to_string(), }); } - fragments.push(TextSnippetFragment { - kind: TextSnippetFragmentKind::Highlighted, + fragments.push(HighlightedFragment { + kind: HighlightedKind::Highlighted, text: self.fragment[range.start..range.end].to_string(), }); @@ -140,8 +114,8 @@ impl SnippetBuilder { } if last_end < self.fragment.len() { - fragments.push(TextSnippetFragment { - kind: TextSnippetFragmentKind::Normal, + fragments.push(HighlightedFragment { + kind: HighlightedKind::Normal, text: self.fragment[last_end..].to_string(), }); } @@ -301,7 +275,7 @@ fn snippet_string( && snip .fragments .iter() - .any(|f| f.kind == TextSnippetFragmentKind::Highlighted) + .any(|f| f.kind == HighlightedKind::Highlighted) { return snip; } @@ -327,8 +301,8 @@ pub fn generate(query: &Query, text: &str, region: &Region, config: SnippetConfi if text.is_empty() { return TextSnippet { - fragments: vec![TextSnippetFragment { - kind: TextSnippetFragmentKind::Normal, + fragments: vec![HighlightedFragment { + kind: HighlightedKind::Normal, text: "".to_string(), }], }; @@ -373,9 +347,9 @@ Survey in 2016, 2017, and 2018."#; text.fragments .into_iter() - .map(|TextSnippetFragment { kind, text }| match kind { - TextSnippetFragmentKind::Normal => text, - TextSnippetFragmentKind::Highlighted => { + .map(|HighlightedFragment { kind, text }| match kind { + HighlightedKind::Normal => text, + HighlightedKind::Highlighted => { format!("{HIGHLIGHTEN_PREFIX}{}{HIGHLIGHTEN_POSTFIX}", text) } }) diff --git a/frontend/.eslintignore b/frontend/.eslintignore index 38972655..0fda2a33 100644 --- a/frontend/.eslintignore +++ b/frontend/.eslintignore @@ -11,3 +11,6 @@ node_modules pnpm-lock.yaml package-lock.json yarn.lock + +# Ignore auto-generated api stubs +src/lib/api/index.ts diff --git a/frontend/.eslintrc.cjs b/frontend/.eslintrc.cjs index 7430e825..291263ad 100644 --- a/frontend/.eslintrc.cjs +++ b/frontend/.eslintrc.cjs @@ -6,6 +6,12 @@ module.exports = { 'plugin:svelte/recommended', 'prettier', ], + rules: { + '@typescript-eslint/no-unused-vars': [ + 'error', + { argsIgnorePattern: '^_', varsIgnorePattern: '^_' }, + ], + }, parser: '@typescript-eslint/parser', plugins: ['@typescript-eslint'], parserOptions: { diff --git a/frontend/package.json b/frontend/package.json index b3c11528..fdf318d5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,7 +10,7 @@ "build": "vite build", "preview": "vite preview", "test": "npm run test:integration && npm run test:unit", - "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", + "check": "svelte-kit sync && svelte-check --fail-on-warnings --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "lint": "prettier --plugin prettier-plugin-svelte --plugin prettier-plugin-tailwindcss --check . && eslint .", "format": "prettier --plugin prettier-plugin-svelte --plugin prettier-plugin-tailwindcss --write .", diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts index 81b7e300..604e7d1e 100644 --- a/frontend/src/lib/api/index.ts +++ b/frontend/src/lib/api/index.ts @@ -297,8 +297,14 @@ export type FullEdge = { label: string; to: Node; }; +export type HighlightedFragment = { + kind: HighlightedKind; + text: string; +}; +export type HighlightedKind = 'normal' | 'highlighted'; +export const HIGHLIGHTED_KINDS = ['normal', 'highlighted'] satisfies HighlightedKind[]; export type HighlightedSpellCorrection = { - highlighted: string; + highlighted: HighlightedFragment[]; raw: string; }; export type HostRankings = { @@ -374,21 +380,12 @@ export type StackOverflowQuestion = { body: CodeOrText[]; }; export type Suggestion = { - highlighted: string; + highlighted: HighlightedFragment[]; raw: string; }; export type TextSnippet = { - fragments: TextSnippetFragment[]; -}; -export type TextSnippetFragment = { - kind: TextSnippetFragmentKind; - text: string; + fragments: HighlightedFragment[]; }; -export type TextSnippetFragmentKind = 'normal' | 'highlighted'; -export const TEXT_SNIPPET_FRAGMENT_KINDS = [ - 'normal', - 'highlighted', -] satisfies TextSnippetFragmentKind[]; export type ThesaurusWidget = { meanings: PartOfSpeechMeaning[]; term: Lemma; diff --git a/frontend/src/lib/components/Searchbar.svelte b/frontend/src/lib/components/Searchbar.svelte index 56516e7f..90908ebe 100644 --- a/frontend/src/lib/components/Searchbar.svelte +++ b/frontend/src/lib/components/Searchbar.svelte @@ -1,7 +1,7 @@ @@ -161,9 +159,13 @@ > - {@html splitAtOverlap(s)[0]}{@html splitAtOverlap(s)[1]} + {#each s as fragment} + {#if fragment.kind == 'highlighted'} + {fragment.text} + {:else} + {fragment.text} + {/if} + {/each} {/each} diff --git a/frontend/src/lib/components/Select.svelte b/frontend/src/lib/components/Select.svelte index eac6d2f6..08292187 100644 --- a/frontend/src/lib/components/Select.svelte +++ b/frontend/src/lib/components/Select.svelte @@ -1,21 +1,23 @@ diff --git a/frontend/src/lib/rankings.ts b/frontend/src/lib/rankings.ts index 54b98e49..d5950992 100644 --- a/frontend/src/lib/rankings.ts +++ b/frontend/src/lib/rankings.ts @@ -26,7 +26,7 @@ export const rankingsToRanked = (rankings: SiteRankings): RankedSites => { export const rankedToRankings = (ranked: RankedSites): SiteRankings => { const result: SiteRankings = {}; - for (const [_, ranking] of Object.entries(Ranking)) { + for (const [, ranking] of Object.entries(Ranking)) { for (const site of ranked[ranking]) { result[site] = ranking; } diff --git a/frontend/src/routes/about/+page.svelte b/frontend/src/routes/about/+page.svelte index 3c1c9a3a..5f2a6b93 100644 --- a/frontend/src/routes/about/+page.svelte +++ b/frontend/src/routes/about/+page.svelte @@ -9,6 +9,7 @@
+ {@html data.md}
diff --git a/frontend/src/routes/privacy-and-happy-lawyers/+page.svelte b/frontend/src/routes/privacy-and-happy-lawyers/+page.svelte index 944b674f..5d9e5979 100644 --- a/frontend/src/routes/privacy-and-happy-lawyers/+page.svelte +++ b/frontend/src/routes/privacy-and-happy-lawyers/+page.svelte @@ -6,5 +6,6 @@
+ {@html data.md}
diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelte index 4ca09a76..bd7e9a8f 100644 --- a/frontend/src/routes/search/+page.svelte +++ b/frontend/src/routes/search/+page.svelte @@ -112,7 +112,13 @@ {@html results.spellCorrection.highlighted}{#each results.spellCorrection.highlighted as frag} + {#if frag.kind == 'highlighted'} + {frag.text} + {:else} + {frag.text} + {/if} + {/each}
diff --git a/frontend/src/routes/search/DirectAnswer.svelte b/frontend/src/routes/search/DirectAnswer.svelte deleted file mode 100644 index 8210a27f..00000000 --- a/frontend/src/routes/search/DirectAnswer.svelte +++ /dev/null @@ -1,26 +0,0 @@ - - -
-
{directAnswer.answer}
-
{@html directAnswer.snippet}
-
- - {directAnswer.prettyUrl} - -
- - {directAnswer.title} - -
-
-
diff --git a/frontend/src/routes/search/Entity.svelte b/frontend/src/routes/search/Entity.svelte index 28b96dee..3ee60b62 100644 --- a/frontend/src/routes/search/Entity.svelte +++ b/frontend/src/routes/search/Entity.svelte @@ -45,7 +45,7 @@
{#each entity.info as [key, value]} -
{@html key}
+
{key}
diff --git a/frontend/src/routes/webmasters/+page.svelte b/frontend/src/routes/webmasters/+page.svelte index 944b674f..5d9e5979 100644 --- a/frontend/src/routes/webmasters/+page.svelte +++ b/frontend/src/routes/webmasters/+page.svelte @@ -6,5 +6,6 @@
+ {@html data.md}
diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index b389883f..ecc92f1a 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -10,7 +10,8 @@ "skipLibCheck": true, "sourceMap": true, "strict": true - } + }, + "exclude": ["svelte.config.js"] // Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias // // If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes diff --git a/scripts/ci/check b/scripts/ci/check index 0312e07a..47f57e4f 100755 --- a/scripts/ci/check +++ b/scripts/ci/check @@ -4,5 +4,5 @@ set -e cargo check cargo check --no-default-features -# skip frontend check until https://github.com/sveltejs/kit/issues/11906 is fixed -# cd frontend && npm run wasm && npm install && npm run check +cd crates/client-wasm && wasm-pack build --target web && cd - +cd frontend && npm install && npm run check && npm run lint