From 976531df57ea4ce9f8f3fb6f3e8fed5488a1d46e Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Mon, 16 Dec 2024 13:41:14 -0800 Subject: [PATCH] fix(docs): Api ref linking in codeblocks (#7383) --- docs/core_docs/.gitignore | 88 +++++++++++---------- docs/core_docs/scripts/code-block-loader.js | 45 ++++++++--- 2 files changed, 77 insertions(+), 56 deletions(-) diff --git a/docs/core_docs/.gitignore b/docs/core_docs/.gitignore index 235b78c4b8e2..01a0e487fd41 100644 --- a/docs/core_docs/.gitignore +++ b/docs/core_docs/.gitignore @@ -54,6 +54,8 @@ docs/tutorials/classification.md docs/tutorials/classification.mdx docs/tutorials/chatbot.md docs/tutorials/chatbot.mdx +docs/concepts/t.md +docs/concepts/t.mdx docs/how_to/trim_messages.md docs/how_to/trim_messages.mdx docs/how_to/tools_prompting.md @@ -216,30 +218,14 @@ docs/how_to/assign.md docs/how_to/assign.mdx docs/how_to/agent_executor.md docs/how_to/agent_executor.mdx -docs/concepts/t.md -docs/concepts/t.mdx -docs/troubleshooting/errors/INVALID_TOOL_RESULTS.md -docs/troubleshooting/errors/INVALID_TOOL_RESULTS.mdx docs/versions/migrating_memory/conversation_summary_memory.md docs/versions/migrating_memory/conversation_summary_memory.mdx docs/versions/migrating_memory/conversation_buffer_window_memory.md docs/versions/migrating_memory/conversation_buffer_window_memory.mdx docs/versions/migrating_memory/chat_history.md docs/versions/migrating_memory/chat_history.mdx -docs/integrations/tools/tavily_search.md -docs/integrations/tools/tavily_search.mdx -docs/integrations/tools/serpapi.md -docs/integrations/tools/serpapi.mdx -docs/integrations/tools/exa_search.md -docs/integrations/tools/exa_search.mdx -docs/integrations/tools/duckduckgo_search.md -docs/integrations/tools/duckduckgo_search.mdx -docs/integrations/toolkits/vectorstore.md -docs/integrations/toolkits/vectorstore.mdx -docs/integrations/toolkits/sql.md -docs/integrations/toolkits/sql.mdx -docs/integrations/toolkits/openapi.md -docs/integrations/toolkits/openapi.mdx +docs/troubleshooting/errors/INVALID_TOOL_RESULTS.md +docs/troubleshooting/errors/INVALID_TOOL_RESULTS.mdx docs/integrations/vectorstores/weaviate.md docs/integrations/vectorstores/weaviate.mdx docs/integrations/vectorstores/upstash.md @@ -266,20 +252,22 @@ docs/integrations/vectorstores/elasticsearch.md docs/integrations/vectorstores/elasticsearch.mdx docs/integrations/vectorstores/chroma.md docs/integrations/vectorstores/chroma.mdx -docs/integrations/stores/in_memory.md -docs/integrations/stores/in_memory.mdx -docs/integrations/stores/file_system.md -docs/integrations/stores/file_system.mdx -docs/integrations/retrievers/tavily.md -docs/integrations/retrievers/tavily.mdx -docs/integrations/retrievers/kendra-retriever.md -docs/integrations/retrievers/kendra-retriever.mdx -docs/integrations/retrievers/exa.md -docs/integrations/retrievers/exa.mdx -docs/integrations/retrievers/bm25.md -docs/integrations/retrievers/bm25.mdx -docs/integrations/retrievers/bedrock-knowledge-bases.md -docs/integrations/retrievers/bedrock-knowledge-bases.mdx +docs/integrations/tools/tavily_search.md +docs/integrations/tools/tavily_search.mdx +docs/integrations/tools/serpapi.md +docs/integrations/tools/serpapi.mdx +docs/integrations/tools/google_scholar.md +docs/integrations/tools/google_scholar.mdx +docs/integrations/tools/exa_search.md +docs/integrations/tools/exa_search.mdx +docs/integrations/tools/duckduckgo_search.md +docs/integrations/tools/duckduckgo_search.mdx +docs/integrations/toolkits/vectorstore.md +docs/integrations/toolkits/vectorstore.mdx +docs/integrations/toolkits/sql.md +docs/integrations/toolkits/sql.mdx +docs/integrations/toolkits/openapi.md +docs/integrations/toolkits/openapi.mdx docs/integrations/text_embedding/togetherai.md docs/integrations/text_embedding/togetherai.mdx docs/integrations/text_embedding/pinecone.md @@ -306,6 +294,20 @@ docs/integrations/text_embedding/bedrock.md docs/integrations/text_embedding/bedrock.mdx docs/integrations/text_embedding/azure_openai.md docs/integrations/text_embedding/azure_openai.mdx +docs/integrations/stores/in_memory.md +docs/integrations/stores/in_memory.mdx +docs/integrations/stores/file_system.md +docs/integrations/stores/file_system.mdx +docs/integrations/retrievers/tavily.md +docs/integrations/retrievers/tavily.mdx +docs/integrations/retrievers/kendra-retriever.md +docs/integrations/retrievers/kendra-retriever.mdx +docs/integrations/retrievers/exa.md +docs/integrations/retrievers/exa.mdx +docs/integrations/retrievers/bm25.md +docs/integrations/retrievers/bm25.mdx +docs/integrations/retrievers/bedrock-knowledge-bases.md +docs/integrations/retrievers/bedrock-knowledge-bases.mdx docs/integrations/llms/together.md docs/integrations/llms/together.mdx docs/integrations/llms/openai.md @@ -384,16 +386,6 @@ docs/integrations/retrievers/self_query/hnswlib.md docs/integrations/retrievers/self_query/hnswlib.mdx docs/integrations/retrievers/self_query/chroma.md docs/integrations/retrievers/self_query/chroma.mdx -docs/integrations/document_loaders/file_loaders/unstructured.md -docs/integrations/document_loaders/file_loaders/unstructured.mdx -docs/integrations/document_loaders/file_loaders/text.md -docs/integrations/document_loaders/file_loaders/text.mdx -docs/integrations/document_loaders/file_loaders/pdf.md -docs/integrations/document_loaders/file_loaders/pdf.mdx -docs/integrations/document_loaders/file_loaders/directory.md -docs/integrations/document_loaders/file_loaders/directory.mdx -docs/integrations/document_loaders/file_loaders/csv.md -docs/integrations/document_loaders/file_loaders/csv.mdx docs/integrations/document_loaders/web_loaders/web_puppeteer.md docs/integrations/document_loaders/web_loaders/web_puppeteer.mdx docs/integrations/document_loaders/web_loaders/web_cheerio.md @@ -405,4 +397,14 @@ docs/integrations/document_loaders/web_loaders/pdf.mdx docs/integrations/document_loaders/web_loaders/langsmith.md docs/integrations/document_loaders/web_loaders/langsmith.mdx docs/integrations/document_loaders/web_loaders/firecrawl.md -docs/integrations/document_loaders/web_loaders/firecrawl.mdx \ No newline at end of file +docs/integrations/document_loaders/web_loaders/firecrawl.mdx +docs/integrations/document_loaders/file_loaders/unstructured.md +docs/integrations/document_loaders/file_loaders/unstructured.mdx +docs/integrations/document_loaders/file_loaders/text.md +docs/integrations/document_loaders/file_loaders/text.mdx +docs/integrations/document_loaders/file_loaders/pdf.md +docs/integrations/document_loaders/file_loaders/pdf.mdx +docs/integrations/document_loaders/file_loaders/directory.md +docs/integrations/document_loaders/file_loaders/directory.mdx +docs/integrations/document_loaders/file_loaders/csv.md +docs/integrations/document_loaders/file_loaders/csv.mdx \ No newline at end of file diff --git a/docs/core_docs/scripts/code-block-loader.js b/docs/core_docs/scripts/code-block-loader.js index f023faf02eb3..23bfcdd13df2 100644 --- a/docs/core_docs/scripts/code-block-loader.js +++ b/docs/core_docs/scripts/code-block-loader.js @@ -5,6 +5,16 @@ const swc = require("@swc/core"); const path = require("path"); const fs = require("fs"); +// Directories generated inside the API docs (excluding "modules"). +const CATEGORIES = [ + "classes", + "enums", + "functions", + "interfaces", + "types", + "variables", +]; + /** * Edge cases where the import will not match the proper API ref path. * This is typically caused by a re-export, or an aliased export so we @@ -90,15 +100,6 @@ const SYMBOLS_TO_SKIP_MAP = { async function webpackLoader(content, map, meta) { const cb = this.async(); const BASE_URL = "https://api.js.langchain.com"; - // Directories generated inside the API docs (excluding "modules"). - const CATEGORIES = [ - "classes", - "enums", - "functions", - "interfaces", - "types", - "variables", - ]; if (!this.resourcePath.endsWith(".ts")) { cb(null, JSON.stringify({ content, imports: [] }), map, meta); @@ -177,12 +178,13 @@ async function webpackLoader(content, map, meta) { let cleanedSource = ""; if (source.startsWith("@langchain/")) { cleanedSource = source - .replace("@langchain/", "langchain_") - .replaceAll("/", "_") - .replaceAll("-", "_"); + .replace("@langchain/", "_langchain_") + .replace(/(?<=_langchain_[^/]+)\//, ".") + .replaceAll(/\//g, "_") + .replaceAll(/-/g, "_"); } else if (source.startsWith("langchain")) { cleanedSource = source - .replace("langchain/", "langchain_") + .replace("langchain/", "langchain.") .replaceAll("/", "_") .replaceAll("-", "_"); } else { @@ -191,6 +193,7 @@ async function webpackLoader(content, map, meta) { ); } const componentPath = `${cleanedSource}.${imported}.html`; + const componentIndexPath = `${cleanedSource}.index.${imported}.html`; /** * Defaults to null, reassigned to string if a match is found. @@ -202,16 +205,31 @@ async function webpackLoader(content, map, meta) { return; } const fullPath = `${category}/${componentPath}`; + const fullIndexPath = `${category}/${componentIndexPath}`; + const pathExists = fs.existsSync(getDocsPath(fullPath)); if (pathExists) { actualPath = fullPath; + return; + } + const indexPathExists = fs.existsSync(getDocsPath(fullIndexPath)); + if (indexPathExists) { + actualPath = fullIndexPath; } }); + return actualPath; }; imports.forEach((imp) => { const { imported, source } = imp; + + if (source.startsWith("@langchain/langgraph")) { + // TODO: Add support for verifying LangGraph API reference links so we can use exact URLs. + imp.docs = "https://langchain-ai.github.io/langgraphjs/reference/"; + return; + } + const apiRefPath = findApiRefPath(imported, source); if (apiRefPath) { @@ -222,6 +240,7 @@ async function webpackLoader(content, map, meta) { const cleanedResourcePath = this.resourcePath.includes("examples/") ? this.resourcePath.split("examples/")[1] : this.resourcePath; + console.warn( { imported,