From bda4bb018495e87b0289ea383003e1b4a6791f6e Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 2 Oct 2024 01:01:40 +0200 Subject: [PATCH 01/11] Try disabling pretraining tests to probe windows ci failure (#13646) --- .../{test_pretraining.py => test_pretraining.py.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename spacy/tests/training/{test_pretraining.py => test_pretraining.py.disabled} (100%) diff --git a/spacy/tests/training/test_pretraining.py b/spacy/tests/training/test_pretraining.py.disabled similarity index 100% rename from spacy/tests/training/test_pretraining.py rename to spacy/tests/training/test_pretraining.py.disabled From 10a6f508ab624cacdaa3443ec84284bd236739ab Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Fri, 11 Oct 2024 11:19:10 +0200 Subject: [PATCH 02/11] Fix landing banner links [ci skip] --- website/src/styles/landing.module.sass | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/src/styles/landing.module.sass b/website/src/styles/landing.module.sass index 5c2a0754bb1..6a703194c0f 100644 --- a/website/src/styles/landing.module.sass +++ b/website/src/styles/landing.module.sass @@ -87,6 +87,9 @@ margin-bottom: 0 height: 100% + a, a:hover + color: inherit + .banner-content-small display: block margin-bottom: 0 !important From 52a4cb0d148f80838277bc7435d64f32b81b4a06 Mon Sep 17 00:00:00 2001 From: sam rxh <118240781+samrxh@users.noreply.github.com> Date: Fri, 11 Oct 2024 04:20:34 -0500 Subject: [PATCH 03/11] Fix 'issue template' link in CONTRIBUTING.md (#13587) [ci skip] --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed75e1fd8bd..9407881a177 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,7 +35,7 @@ so that more people can benefit from it. When opening an issue, use a **descriptive title** and include your **environment** (operating system, Python version, spaCy version). Our -[issue template](https://github.com/explosion/spaCy/issues/new) helps you +[issue templates](https://github.com/explosion/spaCy/issues/new/choose) help you remember the most important details to include. If you've discovered a bug, you can also submit a [regression test](#fixing-bugs) straight away. When you're opening an issue to report the bug, simply refer to your pull request in the From 44d190645307b331649c3081f0a294c6ad0d264d Mon Sep 17 00:00:00 2001 From: aravind-mc Date: Fri, 11 Oct 2024 14:51:57 +0530 Subject: [PATCH 04/11] Update universe.json to add my spaCy online course (#13632) [ci skip] --- website/meta/universe.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index 45b3f625c62..92829bc56e6 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -2587,6 +2587,20 @@ "courses" ] }, + { + "type": "education", + "id": "spacy-quickstart", + "title": "spaCy Quickstart", + "slogan": "Learn spaCy basics quickly by visualizing various Doc objects", + "description":"In this course, I use the itables Python library inside a Jupyter notebook so that you can visualize the different spaCy document objects. This will provide a solid foundation for people who wish to learn the spaCy NLP library." + "url": "https://learnspacy.com/courses/spacy-quickstart/", + "image": "https://learnspacy.com/wp-content/uploads/2024/09/custom_search_builder_spacy-2048x1202.png", + "thumb": "https://learnspacy.com/wp-content/uploads/2024/09/learnspacy_logo.png", + "author": "Aravind Mohanoor", + "category": [ + "courses" + ] + }, { "type": "education", "id": "video-spacys-ner-model", From 8d2902b0e72d0c6bb644fc6243e95d8487d12574 Mon Sep 17 00:00:00 2001 From: "Andrei (Andrey) Khropov" Date: Fri, 11 Oct 2024 11:23:12 +0200 Subject: [PATCH 05/11] Fix misspelling (#13631) [ci skip] --- spacy/lang/hr/lemma_lookup_license.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/lang/hr/lemma_lookup_license.txt b/spacy/lang/hr/lemma_lookup_license.txt index 04671e4043f..9cc003a1107 100644 --- a/spacy/lang/hr/lemma_lookup_license.txt +++ b/spacy/lang/hr/lemma_lookup_license.txt @@ -1,5 +1,5 @@ The list of Croatian lemmas was extracted from the reldi-tagger repository (https://github.com/clarinsi/reldi-tagger). -Reldi-tagger is licesned under the Apache 2.0 licence. +Reldi-tagger is licensed under the Apache 2.0 licence. @InProceedings{ljubesic16-new, author = {Nikola Ljubešić and Filip Klubička and Željko Agić and Ivo-Pavao Jazbec}, @@ -12,4 +12,4 @@ Reldi-tagger is licesned under the Apache 2.0 licence. publisher = {European Language Resources Association (ELRA)}, address = {Paris, France}, isbn = {978-2-9517408-9-1} - } \ No newline at end of file + } From ae5c3e078dd5326dddf4cbd6cd2f102f56b22267 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Fri, 11 Oct 2024 11:24:42 +0200 Subject: [PATCH 06/11] Fix universe.json [ci skip] --- website/meta/universe.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index 92829bc56e6..918a6db01cc 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -2592,7 +2592,7 @@ "id": "spacy-quickstart", "title": "spaCy Quickstart", "slogan": "Learn spaCy basics quickly by visualizing various Doc objects", - "description":"In this course, I use the itables Python library inside a Jupyter notebook so that you can visualize the different spaCy document objects. This will provide a solid foundation for people who wish to learn the spaCy NLP library." + "description": "In this course, I use the itables Python library inside a Jupyter notebook so that you can visualize the different spaCy document objects. This will provide a solid foundation for people who wish to learn the spaCy NLP library.", "url": "https://learnspacy.com/courses/spacy-quickstart/", "image": "https://learnspacy.com/wp-content/uploads/2024/09/custom_search_builder_spacy-2048x1202.png", "thumb": "https://learnspacy.com/wp-content/uploads/2024/09/learnspacy_logo.png", From 0d7e57fc3e57acfeeadf6f4291b2d587dccac247 Mon Sep 17 00:00:00 2001 From: thjbdvlt <109964512+thjbdvlt@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:26:15 +0200 Subject: [PATCH 07/11] universe-pipeline-solipCysme-french (#13627) [ci skip] --- website/meta/universe.json | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index 918a6db01cc..b3542379012 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -276,6 +276,47 @@ "ancient Greek" ] }, + { + "id": "solipcysme", + "title": "solipCysme", + "slogan": "spaCy pipeline for french fictions and first person point of view texts.", + "description": "__solipCysme__ is a pipeline for french language, designed for the analysis of fictions and first person point of view texts, with a focus on personal pronouns.", + "github": "thjbdvlt/solipCysme", + "code_example": [ + "pip install https://huggingface.co/thjbdvlt/fr_solipcysme/resolve/main/fr_solipcysme-any-py3-none-any.whl", + "", + "import spacy", + "", + "nlp = spacy.load('fr_solipcysme')", + "for i in nlp(", + "'la MACHINE à (b)rouiller le temps s'est peut-être déraillée..?'", + "):", + " print(", + " i, ", + " i.norm_, ", + " i.pos_, ", + " i.morph, ", + " i.lemma_, ", + " i.dep_, ", + " i._.tokentype,", + " i._.vv_pos,", + " i._.vv_morph", + " )" + ], + "code_language": "python", + "author": "thjbdvlt", + "author_links": { + "github": "thjbdvlt" + }, + "category": [ + "pipeline", + "research", + "models" + ], + "tags": [ + "french" + ] + }, { "id": "spacy-cleaner", "title": "spacy-cleaner", From 1ee9a1905972055d3760adbc9f7667fdeca9a081 Mon Sep 17 00:00:00 2001 From: Sergei Pashakhin Date: Wed, 23 Oct 2024 10:06:36 +0000 Subject: [PATCH 08/11] Fix typo (#13657) [ci skip] --- website/docs/api/large-language-models.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/api/large-language-models.mdx b/website/docs/api/large-language-models.mdx index cefd5c66ee1..6e2436cc138 100644 --- a/website/docs/api/large-language-models.mdx +++ b/website/docs/api/large-language-models.mdx @@ -1597,7 +1597,7 @@ The name of the model to be used has to be passed in via the `name` attribute. | Argument | Description | | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `name` | The name of a mdodel supported by LangChain for this API. ~~str~~ | +| `name` | The name of a model supported by LangChain for this API. ~~str~~ | | `config` | Configuration passed on to the LangChain model. Defaults to `{}`. ~~Dict[Any, Any]~~ | | `query` | Function that executes the prompts. If `None`, defaults to `spacy.CallLangChain.v1`. ~~Optional[Callable[["langchain.llms.BaseLLM", Iterable[Any]], Iterable[Any]]]~~ | From 15fbf5ef36689d925549d96cb9b9469b5140d7bb Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 23 Oct 2024 19:07:01 +0900 Subject: [PATCH 09/11] docs: update rule-based-matching.mdx (#13665) [ci skip] --- website/docs/usage/rule-based-matching.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/usage/rule-based-matching.mdx b/website/docs/usage/rule-based-matching.mdx index e5b98da3a8c..8632f4573eb 100644 --- a/website/docs/usage/rule-based-matching.mdx +++ b/website/docs/usage/rule-based-matching.mdx @@ -720,7 +720,7 @@ matches = matcher(doc) # Serve visualization of sentences containing match with displaCy # set manual=True to make displaCy render straight from a dictionary -# (if you're not running the code within a Jupyer environment, you can +# (if you're not running the code within a Jupyter environment, you can # use displacy.serve instead) displacy.render(matched_sents, style="ent", manual=True) ``` From 3ecec1324c31e8531b50304b59a1972c3e0961b3 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 23 Oct 2024 12:42:54 +0200 Subject: [PATCH 10/11] Usage page on memory management, explaining memory zones and doc_cleaner (#13643) [ci skip] Co-authored-by: Ines Montani --- website/docs/api/language.mdx | 22 ++ website/docs/usage/memory-management.mdx | 131 ++++++++ website/meta/sidebars.json | 376 ++++++++++++++++++----- 3 files changed, 454 insertions(+), 75 deletions(-) create mode 100644 website/docs/usage/memory-management.mdx diff --git a/website/docs/api/language.mdx b/website/docs/api/language.mdx index 068e8ea7885..b969ddc5332 100644 --- a/website/docs/api/language.mdx +++ b/website/docs/api/language.mdx @@ -890,6 +890,28 @@ when loading a config with | `pipe_name` | Name of pipeline component to replace listeners for. ~~str~~ | | `listeners` | The paths to the listeners, relative to the component config, e.g. `["model.tok2vec"]`. Typically, implementations will only connect to one tok2vec component, `model.tok2vec`, but in theory, custom models can use multiple listeners. The value here can either be an empty list to not replace any listeners, or a _complete_ list of the paths to all listener layers used by the model that should be replaced.~~Iterable[str]~~ | +## Language.memory_zone {id="memory_zone",tag="contextmanager",version="3.8"} + +Begin a block where all resources allocated during the block will be freed at +the end of it. If a resources was created within the memory zone block, +accessing it outside the block is invalid. Behavior of this invalid access is +undefined. Memory zones should not be nested. The memory zone is helpful for +services that need to process large volumes of text with a defined memory budget. + +> ```python +> ### Example +> counts = Counter() +> with nlp.memory_zone(): +> for doc in nlp.pipe(texts): +> for token in doc: +> counts[token.text] += 1 +> ``` + +| Name | Description | +| --- | --- | +| `mem` | Optional `cymem.Pool` object to own allocations (created if not provided). This argument is not required for ordinary usage. Defaults to `None`. ~~Optional[cymem.Pool]~~ | +| **RETURNS** | The memory pool that owns the allocations. This object is not required for ordinary usage. ~~Iterator[cymem.Pool]~~ | + ## Language.meta {id="meta",tag="property"} Meta data for the `Language` class, including name, version, data sources, diff --git a/website/docs/usage/memory-management.mdx b/website/docs/usage/memory-management.mdx new file mode 100644 index 00000000000..330c117cd31 --- /dev/null +++ b/website/docs/usage/memory-management.mdx @@ -0,0 +1,131 @@ +--- +title: Memory Management +teaser: Managing Memory for persistent services +version: 3.8 +menu: + - ['Memory Zones', 'memoryzones'] + - ['Clearing Doc attributes', 'doc-attrs'] +--- + +spaCy maintains a few internal caches that improve speed, +but cause memory to increase slightly over time. If you're +running a batch process that you don't need to be long-lived, +the increase in memory usage generally isn't a problem. +However, if you're running spaCy inside a web service, you'll +often want spaCy's memory usage to stay consistent. Transformer +models can also run into memory problems sometimes, especially when +used on a GPU. + +## Memory zones {id="memoryzones"} + +You can tell spaCy to free data from its internal caches (especially the +[`Vocab`](/api/vocab)) using the [`Language.memory_zone`](/api/language#memory_zone) context manager. Enter +the contextmanager and process your text within it, and spaCy will +**reset its internal caches** (freeing up the associated memory) at the +end of the block. spaCy objects created inside the memory zone must +not be accessed once the memory zone is finished. + +```python +### Using memory zones +from collections import Counter + +def count_words(nlp, texts): + counts = Counter() + with nlp.memory_zone(): + for doc in nlp.pipe(texts): + for token in doc: + counts[token.text] += 1 + return counts +``` + + + +Exiting the memory-zone invalidates all `Doc`, `Token`, `Span` and `Lexeme` +objects that were created within it. If you access these objects +after the memory zone exits, you may encounter a segmentation fault +due to invalid memory access. + + + +spaCy needs the memory zone contextmanager because the processing pipeline +can't keep track of which [`Doc`](/api/doc) objects are referring to data in the shared +[`Vocab`](/api/vocab) cache. For instance, when spaCy encounters a new word, a new [`Lexeme`](/api/lexeme) +entry is stored in the `Vocab`, and the `Doc` object points to this shared +data. When the `Doc` goes out of scope, the `Vocab` has no way of knowing that +this `Lexeme` is no longer in use. + +The memory zone solves this problem by +allowing you to tell the processing pipeline that all data created +between two points is no longer in use. It is up to the you to honor +this agreement. If you access objects that are supposed to no longer be in +use, you may encounter a segmentation fault due to invalid memory access. + +A common use case for memory zones will be **within a web service**. The processing +pipeline can be loaded once, either as a context variable or a global, and each +request can be handled within a memory zone: + +```python +### Memory zones with FastAPI {highlight="10,23"} +from fastapi import FastAPI, APIRouter, Depends, Request +import spacy +from spacy.language import Language + +router = APIRouter() + + +def make_app(): + app = FastAPI() + app.state.NLP = spacy.load("en_core_web_sm") + app.include_router(router) + return app + + +def get_nlp(request: Request) -> Language: + return request.app.state.NLP + + +@router.post("/parse") +def parse_texts( + *, text_batch: list[str], nlp: Language = Depends(get_nlp) +) -> list[dict]: + with nlp.memory_zone(): + # Put the spaCy call within a separate function, so we can't + # leak the Doc objects outside the scope of the memory zone. + output = _process_text(nlp, text_batch) + return output + + +def _process_text(nlp: Language, texts: list[str]) -> list[dict]: + # Call spaCy, and transform the output into our own data + # structures. This function is called from inside a memory + # zone, so must not return the spaCy objects. + docs = list(nlp.pipe(texts)) + return [ + { + "tokens": [{"text": t.text} for t in doc], + "entities": [ + {"start": e.start, "end": e.end, "label": e.label_} for e in doc.ents + ], + } + for doc in docs + ] + + +app = make_app() +``` + +## Clearing transformer tensors and other Doc attributes {id="doc-attrs"} + +The [`Transformer`](/api/transformer) and [`Tok2Vec`](/api/tok2vec) components set intermediate values onto the `Doc` +object during parsing. This can cause GPU memory to be exhausted if many `Doc` +objects are kept in memory together. + +To resolve this, you can add the [`doc_cleaner`](/api/pipeline-functions#doc_cleaner) component to your pipeline. By default +this will clean up the [`Doc._.trf_data`](/api/transformer#custom_attributes) extension attribute and the [`Doc.tensor`](/api/doc#attributes) attribute. +You can have it clean up other intermediate extension attributes you use in custom +pipeline components as well. + +```python +### Adding the doc_cleaner +nlp.add_pipe("doc_cleaner", config={"attrs": {"tensor": None}}) +``` diff --git a/website/meta/sidebars.json b/website/meta/sidebars.json index 2df120ffa7b..74a9eac054e 100644 --- a/website/meta/sidebars.json +++ b/website/meta/sidebars.json @@ -5,45 +5,96 @@ { "label": "Get started", "items": [ - { "text": "Installation", "url": "/usage" }, - { "text": "Models & Languages", "url": "/usage/models" }, - { "text": "Facts & Figures", "url": "/usage/facts-figures" }, - { "text": "spaCy 101", "url": "/usage/spacy-101" }, - { "text": "New in v3.7", "url": "/usage/v3-7" }, - { "text": "New in v3.6", "url": "/usage/v3-6" }, - { "text": "New in v3.5", "url": "/usage/v3-5" } + { + "text": "Installation", + "url": "/usage" + }, + { + "text": "Models & Languages", + "url": "/usage/models" + }, + { + "text": "Facts & Figures", + "url": "/usage/facts-figures" + }, + { + "text": "spaCy 101", + "url": "/usage/spacy-101" + }, + { + "text": "New in v3.7", + "url": "/usage/v3-7" + }, + { + "text": "New in v3.6", + "url": "/usage/v3-6" + }, + { + "text": "New in v3.5", + "url": "/usage/v3-5" + } ] }, { "label": "Guides", "items": [ - { "text": "Linguistic Features", "url": "/usage/linguistic-features" }, - { "text": "Rule-based Matching", "url": "/usage/rule-based-matching" }, - { "text": "Processing Pipelines", "url": "/usage/processing-pipelines" }, + { + "text": "Linguistic Features", + "url": "/usage/linguistic-features" + }, + { + "text": "Rule-based Matching", + "url": "/usage/rule-based-matching" + }, + { + "text": "Processing Pipelines", + "url": "/usage/processing-pipelines" + }, { "text": "Embeddings & Transformers", "url": "/usage/embeddings-transformers" }, { "text": "Large Language Models", - "url": "/usage/large-language-models", - "tag": "new" + "url": "/usage/large-language-models" + }, + { + "text": "Training Models", + "url": "/usage/training" }, - { "text": "Training Models", "url": "/usage/training" }, { "text": "Layers & Model Architectures", "url": "/usage/layers-architectures" }, - { "text": "spaCy Projects", "url": "/usage/projects" }, - { "text": "Saving & Loading", "url": "/usage/saving-loading" }, - { "text": "Visualizers", "url": "/usage/visualizers" } + { + "text": "spaCy Projects", + "url": "/usage/projects" + }, + { + "text": "Saving & Loading", + "url": "/usage/saving-loading" + }, + { + "text": "Memory Management", + "url": "/usage/memory-management" + }, + { + "text": "Visualizers", + "url": "/usage/visualizers" + } ] }, { "label": "Resources", "items": [ - { "text": "Project Templates", "url": "https://github.com/explosion/projects" }, - { "text": "v2.x Documentation", "url": "https://v2.spacy.io" }, + { + "text": "Project Templates", + "url": "https://github.com/explosion/projects" + }, + { + "text": "v2.x Documentation", + "url": "https://v2.spacy.io" + }, { "text": "Custom Solutions", "url": "https://explosion.ai/custom-solutions" @@ -57,7 +108,12 @@ "items": [ { "label": "Models", - "items": [{ "text": "Overview", "url": "/models" }] + "items": [ + { + "text": "Overview", + "url": "/models" + } + ] }, { "label": "Trained Pipelines", @@ -71,91 +127,261 @@ { "label": "Overview", "items": [ - { "text": "Library Architecture", "url": "/api" }, - { "text": "Model Architectures", "url": "/api/architectures" }, - { "text": "Data Formats", "url": "/api/data-formats" }, - { "text": "Command Line", "url": "/api/cli" }, - { "text": "Functions", "url": "/api/top-level" } + { + "text": "Library Architecture", + "url": "/api" + }, + { + "text": "Model Architectures", + "url": "/api/architectures" + }, + { + "text": "Data Formats", + "url": "/api/data-formats" + }, + { + "text": "Command Line", + "url": "/api/cli" + }, + { + "text": "Functions", + "url": "/api/top-level" + } ] }, { "label": "Containers", "items": [ - { "text": "Doc", "url": "/api/doc" }, - { "text": "DocBin", "url": "/api/docbin" }, - { "text": "Example", "url": "/api/example" }, - { "text": "Language", "url": "/api/language" }, - { "text": "Lexeme", "url": "/api/lexeme" }, - { "text": "Span", "url": "/api/span" }, - { "text": "SpanGroup", "url": "/api/spangroup" }, - { "text": "Token", "url": "/api/token" } + { + "text": "Doc", + "url": "/api/doc" + }, + { + "text": "DocBin", + "url": "/api/docbin" + }, + { + "text": "Example", + "url": "/api/example" + }, + { + "text": "Language", + "url": "/api/language" + }, + { + "text": "Lexeme", + "url": "/api/lexeme" + }, + { + "text": "Span", + "url": "/api/span" + }, + { + "text": "SpanGroup", + "url": "/api/spangroup" + }, + { + "text": "Token", + "url": "/api/token" + } ] }, { "label": "Pipeline", "items": [ - { "text": "AttributeRuler", "url": "/api/attributeruler" }, - { "text": "CoreferenceResolver", "url": "/api/coref" }, - { "text": "CuratedTransformer", "url": "/api/curatedtransformer" }, - { "text": "DependencyParser", "url": "/api/dependencyparser" }, - { "text": "EditTreeLemmatizer", "url": "/api/edittreelemmatizer" }, - { "text": "EntityLinker", "url": "/api/entitylinker" }, - { "text": "EntityRecognizer", "url": "/api/entityrecognizer" }, - { "text": "EntityRuler", "url": "/api/entityruler" }, - { "text": "Large Language Models", "url": "/api/large-language-models" }, - { "text": "Lemmatizer", "url": "/api/lemmatizer" }, - { "text": "Morphologizer", "url": "/api/morphologizer" }, - { "text": "SentenceRecognizer", "url": "/api/sentencerecognizer" }, - { "text": "Sentencizer", "url": "/api/sentencizer" }, - { "text": "SpanCategorizer", "url": "/api/spancategorizer" }, - { "text": "SpanFinder", "url": "/api/spanfinder" }, - { "text": "SpanResolver", "url": "/api/span-resolver" }, - { "text": "SpanRuler", "url": "/api/spanruler" }, - { "text": "Tagger", "url": "/api/tagger" }, - { "text": "TextCategorizer", "url": "/api/textcategorizer" }, - { "text": "Tok2Vec", "url": "/api/tok2vec" }, - { "text": "Tokenizer", "url": "/api/tokenizer" }, - { "text": "TrainablePipe", "url": "/api/pipe" }, - { "text": "Transformer", "url": "/api/transformer" }, - { "text": "Other Functions", "url": "/api/pipeline-functions" } + { + "text": "AttributeRuler", + "url": "/api/attributeruler" + }, + { + "text": "CoreferenceResolver", + "url": "/api/coref" + }, + { + "text": "CuratedTransformer", + "url": "/api/curatedtransformer" + }, + { + "text": "DependencyParser", + "url": "/api/dependencyparser" + }, + { + "text": "EditTreeLemmatizer", + "url": "/api/edittreelemmatizer" + }, + { + "text": "EntityLinker", + "url": "/api/entitylinker" + }, + { + "text": "EntityRecognizer", + "url": "/api/entityrecognizer" + }, + { + "text": "EntityRuler", + "url": "/api/entityruler" + }, + { + "text": "Large Language Models", + "url": "/api/large-language-models" + }, + { + "text": "Lemmatizer", + "url": "/api/lemmatizer" + }, + { + "text": "Morphologizer", + "url": "/api/morphologizer" + }, + { + "text": "SentenceRecognizer", + "url": "/api/sentencerecognizer" + }, + { + "text": "Sentencizer", + "url": "/api/sentencizer" + }, + { + "text": "SpanCategorizer", + "url": "/api/spancategorizer" + }, + { + "text": "SpanFinder", + "url": "/api/spanfinder" + }, + { + "text": "SpanResolver", + "url": "/api/span-resolver" + }, + { + "text": "SpanRuler", + "url": "/api/spanruler" + }, + { + "text": "Tagger", + "url": "/api/tagger" + }, + { + "text": "TextCategorizer", + "url": "/api/textcategorizer" + }, + { + "text": "Tok2Vec", + "url": "/api/tok2vec" + }, + { + "text": "Tokenizer", + "url": "/api/tokenizer" + }, + { + "text": "TrainablePipe", + "url": "/api/pipe" + }, + { + "text": "Transformer", + "url": "/api/transformer" + }, + { + "text": "Other Functions", + "url": "/api/pipeline-functions" + } ] }, { "label": "Matchers", "items": [ - { "text": "DependencyMatcher", "url": "/api/dependencymatcher" }, - { "text": "Matcher", "url": "/api/matcher" }, - { "text": "PhraseMatcher", "url": "/api/phrasematcher" } + { + "text": "DependencyMatcher", + "url": "/api/dependencymatcher" + }, + { + "text": "Matcher", + "url": "/api/matcher" + }, + { + "text": "PhraseMatcher", + "url": "/api/phrasematcher" + } ] }, { "label": "Other", "items": [ - { "text": "Attributes", "url": "/api/attributes" }, - { "text": "BaseVectors", "url": "/api/basevectors" }, - { "text": "Corpus", "url": "/api/corpus" }, - { "text": "InMemoryLookupKB", "url": "/api/inmemorylookupkb" }, - { "text": "KnowledgeBase", "url": "/api/kb" }, - { "text": "Lookups", "url": "/api/lookups" }, - { "text": "MorphAnalysis", "url": "/api/morphology#morphanalysis" }, - { "text": "Morphology", "url": "/api/morphology" }, - { "text": "Scorer", "url": "/api/scorer" }, - { "text": "StringStore", "url": "/api/stringstore" }, - { "text": "Vectors", "url": "/api/vectors" }, - { "text": "Vocab", "url": "/api/vocab" } + { + "text": "Attributes", + "url": "/api/attributes" + }, + { + "text": "BaseVectors", + "url": "/api/basevectors" + }, + { + "text": "Corpus", + "url": "/api/corpus" + }, + { + "text": "InMemoryLookupKB", + "url": "/api/inmemorylookupkb" + }, + { + "text": "KnowledgeBase", + "url": "/api/kb" + }, + { + "text": "Lookups", + "url": "/api/lookups" + }, + { + "text": "MorphAnalysis", + "url": "/api/morphology#morphanalysis" + }, + { + "text": "Morphology", + "url": "/api/morphology" + }, + { + "text": "Scorer", + "url": "/api/scorer" + }, + { + "text": "StringStore", + "url": "/api/stringstore" + }, + { + "text": "Vectors", + "url": "/api/vectors" + }, + { + "text": "Vocab", + "url": "/api/vocab" + } ] }, { "label": "Cython", "items": [ - { "text": "Architecture", "url": "/api/cython" }, - { "text": "Classes", "url": "/api/cython-classes" }, - { "text": "Structs", "url": "/api/cython-structs" } + { + "text": "Architecture", + "url": "/api/cython" + }, + { + "text": "Classes", + "url": "/api/cython-classes" + }, + { + "text": "Structs", + "url": "/api/cython-structs" + } ] }, { "label": "Legacy", - "items": [{ "text": "Legacy functions", "url": "/api/legacy" }] + "items": [ + { + "text": "Legacy functions", + "url": "/api/legacy" + } + ] } ] } From 3e30b5bef68240a3def1aef3914a9b8523258c00 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Tue, 19 Nov 2024 10:43:40 +0100 Subject: [PATCH 11/11] Add spacy-layout [ci skip] --- website/meta/universe.json | 42 ++++++++++++++++++++++++++++++++++ website/src/templates/index.js | 4 ++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index b3542379012..b65702885de 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1394,6 +1394,48 @@ "website": "https://ines.io" } }, + { + "id": "spacy-layout", + "slogan": "Process PDFs, Word documents and more with spaCy", + "github": "explosion/spacy-layout", + "description": "This plugin integrates with [Docling](https://ds4sd.github.io/docling/) to bring structured processing of PDFs, Word documents and other input formats to your spaCy pipeline. It outputs clean, structured data in a text-based format and outputs spaCy's familiar `Doc` objects that let you access labelled text spans like sections, headings, or footnotes.\n\nThis workflow makes it easy to apply powerful NLP techniques to your documents, including linguistic analysis, named entity recognition, text classification and more. It's also great for implementing chunking for RAG pipelines.", + "pip": "spacy-layout", + "category": [ + "pipeline" + ], + "code_example": [ + "import spacy", + "from spacy_layout import spaCyLayout", + "", + "nlp = spacy.blank(\"en\")", + "layout = spaCyLayout(nlp)", + "", + "# Process a document and create a spaCy Doc object", + "doc = layout(\"./starcraft.pdf\")", + "", + "# The text-based contents of the document", + "print(doc.text)", + "# Document layout including pages and page sizes", + "print(doc._.layout)", + "", + "# Layout spans for different sections", + "for span in doc.spans[\"layout\"]:", + " # Document section and token and character offsets into the text", + " print(span.text, span.start, span.end, span.start_char, span.end_char)", + " # Section type, e.g. \"text\", \"title\", \"section_header\" etc.", + " print(span.label_)", + " # Layout features of the section, including bounding box", + " print(span._.layout)", + " # Closest heading to the span (accuracy depends on document structure)", + " print(span._.heading)" + ], + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + } + }, { "id": "spacyopentapioca", "title": "spaCyOpenTapioca", diff --git a/website/src/templates/index.js b/website/src/templates/index.js index 754cf47bf58..b4c6f8e00ee 100644 --- a/website/src/templates/index.js +++ b/website/src/templates/index.js @@ -58,8 +58,8 @@ const AlertSpace = ({ nightly, legacy }) => { } const navAlert = ( - - 💥 New: Case study with S&P Global + + 💥 New: spaCy for PDFs and Word docs )