feat: check document store and retriever dimensions before calculating embeddings for all documents

.github/ISSUE_TEMPLATE/breaking-change-proposal.md

@@ -0,0 +1,27 @@
+---
+name: Breaking change proposal
+about: Track a breaking change in Haystack
+title: ''
+labels: breaking change
+assignees: ''
+
+---
+
+## Summary and motivation
+
+Briefly explain how the change is breaking and why it is needed.
+
+## Checklist
+
+```[tasklist]
+### Tasks
+- [ ] The changes are merged in the `main` branch (Code + Docstrings)
+- [ ] Release notes have documented the breaking change
+- [ ] A new version of `haystack-ai` has been released on PyPI
+- [ ] Docs at https://docs.haystack.deepset.ai/ were updated
+- [ ] Integrations on [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) were updated (if needed) - This step might require a [Breaking change proposal](https://github.com/deepset-ai/haystack-core-integrations/issues/new?assignees=&labels=breaking+change&projects=&template=breaking-change-proposal.md&title=) on the repo
+- [ ] Notebooks on https://github.com/deepset-ai/haystack-cookbook were updated (if needed)
+- [ ] Tutorials on https://github.com/deepset-ai/haystack-tutorials were updated (if needed)
+- [ ] Articles on https://github.com/deepset-ai/haystack-home/tree/main/content were updated (if needed)
+- [ ] Integration tile on https://github.com/deepset-ai/haystack-integrations was updated (if needed)
+```

.github/labeler.yml

@@ -1,105 +1,40 @@
+# Release lines
+2.x:
+ - base-branch: 'main'
+
+1.x:
+ - base-branch: 'v1.x'
+
 # Proposals
 proposal:
-- proposals/text/*
-
-# 2.x
-2.x:
-- haystack/preview/**/*
-- test/preview/**/*
+- changed-files:
+  - any-glob-to-any-file: proposals/text/*
 
 # Topics
 topic:tests:
-- test/**/*
-- test/*
-- conftest.py
+- changed-files:
+  - any-glob-to-any-file: ['test/**/*','test/*']
+
 topic:docker:
-- docker/*
-- docker-compose.yml
+- changed-files:
+  - any-glob-to-any-file: docker/*
+
 topic:CI:
-- .github/*
-- .github/**/*
+- changed-files:
+  - any-glob-to-any-file: ['.github/*','.github/**/*']
+
 topic:DX:
-- CONTRIBUTING.md
-- .pre-commit-config.yaml
-- .gitignore
-topic:dependencies:
-- pyproject.toml
+- changed-files:
+  - any-glob-to-any-file: ["CONTRIBUTING.md", ".pre-commit-config.yaml",".gitignore"]
+
 topic:build/distribution:
-- pyproject.toml
-topic:modeling:
-- haystack/modeling/*
-- haystack/modeling/**/*
-- test/modeling/*
-topic:reader:
-- haystack/nodes/reader/*
-- test/nodes/test_reader.py
-topic:retriever:
-- haystack/nodes/retriever/*
-- test/nodes/test_retriever.py
-topic:agent:
-- haystack/agents/*
-- test/agents/*
-topic:pipeline:
-- haystack/pipelines/*
-- haystack/nodes/other/*
-- test/pipelines/*
-- test/nodes/test_other.py
-topic:rest_api:
-- rest_api/*
-- rest_api/**/*
-topic:LLM:
-- haystack/nodes/prompt/*
-- test/nodes/test_prompt_node.py
-topic:promptnode:
-- haystack/nodes/prompt/*
-- test/prompt/*
-topic:file_converter:
-- haystack/nodes/file_converter/*
-- test/nodes/test_file_converter.py
-topic:crawler:
-- haystack/nodes/connector/*
-- test/nodes/test_connector.py
-topic:preprocessing:
-- haystack/nodes/preprocessor/*
-- test/nodes/test_preprocessor.py
-topic:images:
-- haystack/nodes/image_to_text/*
-- test/nodes/test_image_to_text.py
-topic:audio:
-- haystack/nodes/audio/*
-- test/nodes/test_audio.py
+- changed-files:
+  - any-glob-to-any-file: pyproject.toml
+
 topic:security:
-- SECURITY.md
-topic:dc-api:
-- haystack/utils/deepsetcloud.py
-topic:document_store:
-- haystack/document_stores/*
-- test/document_stores/*
-topic:weaviate:
-- haystack/document_stores/weaviate.py
-- test/document_stores/test_weaviate.py
-topic:elasticsearch:
-- haystack/document_stores/search_engine.py
-- haystack/document_stores/elasticsearch/*
-- test/document_stores/test_elasticsearch.py
-topic:opensearch:
-- haystack/document_stores/search_engine.py
-- haystack/document_stores/opensearch.py
-- test/document_stores/test_opensearch.py
-topic:sql:
-- haystack/document_stores/sql.py
-- test/document_stores/test_sql.py
-- test/document_stores/test_sql_based.py
-topic:faiss:
-- haystack/document_stores/faiss.py
-- test/document_stores/test_faiss.py
-- test/document_stores/test_sql_based.py
-topic:pinecone:
-- haystack/document_stores/pinecone.py
-- test/document_stores/test_pinecone.py
-topic:knowledge_graph:
-- haystack/document_stores/memory_knowledgegraph.py
-- test/document_stores/test_knowledge_graph.py
-topic:dc_document_store:
-- haystack/document_stores/deepsetcloud.py
-- test/document_stores/test_deepsetcloud.py
+- changed-files:
+  - any-glob-to-any-file: SECURITY.md
+
+topic:core:
+- changed-files:
+  - any-glob-to-any-file: haystack/core/**/*

.github/utils/delete_outdated_docs.py

@@ -0,0 +1,72 @@
+import argparse
+import base64
+import os
+import re
+from pathlib import Path
+from typing import List
+
+import requests
+import yaml
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+
+
+def readme_token():
+    api_key = os.getenv("README_API_KEY", None)
+    if not api_key:
+        raise Exception("README_API_KEY env var is not set")
+
+    api_key = f"{api_key}:"
+    return base64.b64encode(api_key.encode("utf-8")).decode("utf-8")
+
+
+def create_headers(version: str):
+    return {"authorization": f"Basic {readme_token()}", "x-readme-version": version}
+
+
+def get_docs_in_category(category_slug: str, version: str) -> List[str]:
+    """
+    Returns the slugs of all documents in a category for the specific version.
+    """
+    url = f"https://dash.readme.com/api/v1/categories/{category_slug}/docs"
+    headers = create_headers(version)
+    res = requests.get(url, headers=headers, timeout=10)
+    return [doc["slug"] for doc in res.json()]
+
+
+def delete_doc(slug: str, version: str):
+    url = f"https://dash.readme.com/api/v1/docs/{slug}"
+    headers = create_headers(version)
+    res = requests.delete(url, headers=headers, timeout=10)
+    res.raise_for_status()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Delete outdated documentation from Readme.io. "
+        "It will delete all documents that are not present in the current config files."
+    )
+    parser.add_argument(
+        "-c", "--config-path", help="Path to folder containing YAML documentation configs", required=True, type=Path
+    )
+    parser.add_argument("-v", "--version", help="The version that will have its documents deleted", required=True)
+    args = parser.parse_args()
+
+    configs = [yaml.safe_load(c.read_text()) for c in args.config_path.glob("*.yml")]
+
+    remote_docs = {}
+    for config in configs:
+        category_slug = config["renderer"]["category_slug"]
+        if category_slug in remote_docs:
+            continue
+        docs = get_docs_in_category(category_slug, args.version)
+
+        remote_docs[category_slug] = docs
+
+    for config in configs:
+        doc_slug = config["renderer"]["slug"]
+        category_slug = config["renderer"]["category_slug"]
+        if doc_slug in remote_docs[category_slug]:
+            continue
+
+        delete_doc(doc_slug, args.version)

.github/utils/pydoc-markdown.sh

@@ -1,19 +1,10 @@
 #!/bin/bash
 
 set -e   # Fails on any error in the following loop
-export PYTHONPATH=$PWD/docs/pydoc # Make the renderers available to pydoc
 cd docs/pydoc
 rm -rf temp && mkdir temp
 cd temp
 for file in ../config/* ; do
     echo "Converting $file..."
     pydoc-markdown "$file"
 done
-# render preview markdown docs
-cd ..
-rm -rf temp-preview && mkdir temp-preview
-cd temp-preview
-for file in ../config-preview/* ; do
-    echo "Converting $file..."
-    pydoc-markdown "$file"
-done