Skip to content

Commit

Permalink
scripts[major]: Cleanup CLI integration template scripts
Browse files Browse the repository at this point in the history
  • Loading branch information
bracesproul committed Aug 2, 2024
1 parent c1f6c5b commit 589354f
Show file tree
Hide file tree
Showing 13 changed files with 463 additions and 540 deletions.
13 changes: 13 additions & 0 deletions libs/langchain-scripts/src/cli/constants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
export const SIDEBAR_LABEL_PLACEHOLDER = "__sidebar_label__";
export const MODULE_NAME_PLACEHOLDER = "__module_name__";
export const PACKAGE_NAME_PLACEHOLDER = "__package_name__";
export const FULL_IMPORT_PATH_PLACEHOLDER = "__full_import_path__";
export const ENV_VAR_NAME_PLACEHOLDER = "__env_var_name__";

export const API_REF_MODULE_PLACEHOLDER = "__api_ref_module__";
export const API_REF_PACKAGE_PLACEHOLDER = "__api_ref_package__";
export const PYTHON_DOC_URL_PLACEHOLDER = "__python_doc_url__";

export const SERIALIZABLE_PLACEHOLDER = "__serializable__";
export const LOCAL_PLACEHOLDER = "__local__";
export const PY_SUPPORT_PLACEHOLDER = "__py_support__";
190 changes: 85 additions & 105 deletions libs/langchain-scripts/src/cli/docs/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,8 @@ import {
greenText,
redBackground,
} from "../utils/get-input.js";

const PACKAGE_NAME_PLACEHOLDER = "__package_name__";
const PACKAGE_NAME_SHORT_SNAKE_CASE_PLACEHOLDER =
"__package_name_short_snake_case__";
const PACKAGE_NAME_SNAKE_CASE_PLACEHOLDER = "__package_name_snake_case__";
const PACKAGE_NAME_PRETTY_PLACEHOLDER = "__package_name_pretty__";
const PACKAGE_IMPORT_PATH_PLACEHOLDER = "__import_path__";
const MODULE_NAME_PLACEHOLDER = "__ModuleName__";
// This should not be prefixed with `Chat` as it's used for API keys.
const MODULE_NAME_ALL_CAPS_PLACEHOLDER = "__MODULE_NAME_ALL_CAPS__";
import { fetchURLStatus } from "../utils/fetch-url-status.js";
import { SIDEBAR_LABEL_PLACEHOLDER, MODULE_NAME_PLACEHOLDER, PACKAGE_NAME_PLACEHOLDER, FULL_IMPORT_PATH_PLACEHOLDER, ENV_VAR_NAME_PLACEHOLDER, API_REF_MODULE_PLACEHOLDER, API_REF_PACKAGE_PLACEHOLDER, PYTHON_DOC_URL_PLACEHOLDER, LOCAL_PLACEHOLDER, SERIALIZABLE_PLACEHOLDER, PY_SUPPORT_PLACEHOLDER } from "../constants.js";

const TOOL_CALLING_PLACEHOLDER = "__tool_calling__";
const JSON_MODE_PLACEHOLDER = "__json_mode__";
Expand All @@ -26,30 +18,11 @@ const TOKEN_LEVEL_STREAMING_PLACEHOLDER = "__token_level_streaming__";
const TOKEN_USAGE_PLACEHOLDER = "__token_usage__";
const LOGPROBS_PLACEHOLDER = "__logprobs__";

const SERIALIZABLE_PLACEHOLDER = "__serializable__";
const LOCAL_PLACEHOLDER = "__local__";
const PY_SUPPORT_PLACEHOLDER = "__py_support__";

const API_REF_BASE_PACKAGE_URL = `https://api.js.langchain.com/modules/langchain_${PACKAGE_NAME_PLACEHOLDER}.html`;
const API_REF_BASE_MODULE_URL = `https://api.js.langchain.com/classes/langchain_${PACKAGE_NAME_PLACEHOLDER}.${MODULE_NAME_PLACEHOLDER}.html`;

const TEMPLATE_PATH = path.resolve("./src/cli/docs/templates/chat.ipynb");
const INTEGRATIONS_DOCS_PATH = path.resolve(
"../../docs/core_docs/docs/integrations/chat"
);

const fetchAPIRefUrl = async (url: string): Promise<boolean> => {
try {
const res = await fetch(url);
if (res.status !== 200) {
throw new Error(`API Reference URL ${url} not found.`);
}
return true;
} catch (_) {
return false;
}
};

type ExtraFields = {
/**
* If tool calling is true, structured output will also be true.
Expand All @@ -65,9 +38,14 @@ type ExtraFields = {
local: boolean;
serializable: boolean;
pySupport: boolean;
envVarName: string;
fullImportPath: string;
packageName: string;
};

async function promptExtraFields(): Promise<ExtraFields> {
async function promptExtraFields(fields: {
envVarGuess: string;
}): Promise<ExtraFields> {
const hasToolCalling = await getUserInput(
"Does this integration support tool calling? (y/n) ",
undefined,
Expand Down Expand Up @@ -124,6 +102,46 @@ async function promptExtraFields(): Promise<ExtraFields> {
true
);

const importPath = await getUserInput(
"What is the full import path of the integration? (e.g @langchain/community/chat_models/togetherai) ",
undefined,
true
);

let packageName = "";
if (importPath.startsWith("langchain/")) {
packageName = "langchain";
} else {
packageName = importPath.split("/").slice(0, 2).join("/");
}

const verifyPackageName = await getUserInput(
`Is ${packageName} the correct package name? (y/n) `,
undefined,
true
);
if (verifyPackageName.toLowerCase() === "n") {
packageName = await getUserInput(
"Please enter the full package name (e.g @langchain/community) ",
undefined,
true
);
}

const isEnvGuessCorrect = await getUserInput(
`Is the environment variable for the API key named ${fields.envVarGuess}? (y/n) `,
undefined,
true
);
let envVarName = fields.envVarGuess;
if (isEnvGuessCorrect.toLowerCase() === "n") {
envVarName = await getUserInput(
"Please enter the correct environment variable name ",
undefined,
true
);
}

return {
toolCalling: hasToolCalling.toLowerCase() === "y",
jsonMode: hasJsonMode.toLowerCase() === "y",
Expand All @@ -136,86 +154,50 @@ async function promptExtraFields(): Promise<ExtraFields> {
local: hasLocal.toLowerCase() === "y",
serializable: hasSerializable.toLowerCase() === "y",
pySupport: hasPySupport.toLowerCase() === "y",
envVarName,
fullImportPath: importPath,
packageName,
};
}

export async function fillChatIntegrationDocTemplate(fields: {
packageName: string;
moduleName: string;
isCommunity: boolean;
className: string;
}) {
// Ask the user if they'd like to fill in extra fields, if so, prompt them.
let extraFields: ExtraFields | undefined;
const shouldPromptExtraFields = await getUserInput(
"Would you like to fill out optional fields? (y/n) ",
"white_background"
);
if (shouldPromptExtraFields.toLowerCase() === "y") {
extraFields = await promptExtraFields();
}

let formattedApiRefPackageUrl = "";
let formattedApiRefModuleUrl = "";
if (fields.isCommunity) {
formattedApiRefPackageUrl = API_REF_BASE_PACKAGE_URL.replace(
PACKAGE_NAME_PLACEHOLDER,
`community_chat_models_${fields.packageName}`
);
formattedApiRefModuleUrl = API_REF_BASE_MODULE_URL.replace(
PACKAGE_NAME_PLACEHOLDER,
`community_chat_models_${fields.packageName}`
).replace(MODULE_NAME_PLACEHOLDER, fields.moduleName);
} else {
formattedApiRefPackageUrl = API_REF_BASE_PACKAGE_URL.replace(
PACKAGE_NAME_PLACEHOLDER,
fields.packageName
);
formattedApiRefModuleUrl = API_REF_BASE_MODULE_URL.replace(
PACKAGE_NAME_PLACEHOLDER,
fields.packageName
).replace(MODULE_NAME_PLACEHOLDER, fields.moduleName);
}

const success = await Promise.all([
fetchAPIRefUrl(formattedApiRefPackageUrl),
fetchAPIRefUrl(formattedApiRefModuleUrl),
const sidebarLabel = fields.className.replace("Chat", "");
const pyDocUrl = `https://python.langchain.com/v0.2/docs/integrations/chat/${sidebarLabel.toLowerCase()}/`;
let envVarName = `${sidebarLabel.toUpperCase()}_API_KEY`;
const extraFields = await promptExtraFields({
envVarGuess: envVarName,
});
envVarName = extraFields.envVarName;

const apiRefModuleUrl = `https://api.js.langchain.com/classes/${extraFields.fullImportPath
.replace("@", "")
.replaceAll("/", "_")
.replaceAll("-", "_")}.${fields.className}.html`;
const apiRefPackageUrl = apiRefModuleUrl
.replace("/classes/", "/modules/")
.replace(`.${fields.className}.html`, ".html");

const apiRefUrlSuccesses = await Promise.all([
fetchURLStatus(apiRefModuleUrl),
fetchURLStatus(apiRefPackageUrl),
]);
if (success.some((s) => s === false)) {
// Don't error out because this might be used before the package is released.
console.error("Invalid package or module name. API reference not found.");
}

const packageNameShortSnakeCase = fields.packageName.replaceAll("-", "_");
let fullPackageNameSnakeCase = "";
let packageNamePretty = "";
let fullPackageImportPath = "";

if (fields.isCommunity) {
fullPackageNameSnakeCase = `langchain_community_chat_models_${packageNameShortSnakeCase}`;
fullPackageImportPath = `@langchain/community/chat_models/${fields.packageName}`;
packageNamePretty = "@langchain/community";
} else {
fullPackageNameSnakeCase = `langchain_${packageNameShortSnakeCase}`;
packageNamePretty = `@langchain/${fields.packageName}`;
fullPackageImportPath = packageNamePretty;
}

let moduleNameAllCaps = fields.moduleName.toUpperCase();
if (moduleNameAllCaps.startsWith("CHAT")) {
moduleNameAllCaps = moduleNameAllCaps.replace("CHAT", "");
if (apiRefUrlSuccesses.find((s) => !s)) {
console.warn(
"API ref URLs invalid. Please manually ensure they are correct."
);
}

const docTemplate = (await fs.promises.readFile(TEMPLATE_PATH, "utf-8"))
.replaceAll(PACKAGE_NAME_PLACEHOLDER, fields.packageName)
.replaceAll(PACKAGE_NAME_SNAKE_CASE_PLACEHOLDER, fullPackageNameSnakeCase)
.replaceAll(
PACKAGE_NAME_SHORT_SNAKE_CASE_PLACEHOLDER,
packageNameShortSnakeCase
)
.replaceAll(PACKAGE_NAME_PRETTY_PLACEHOLDER, packageNamePretty)
.replaceAll(PACKAGE_IMPORT_PATH_PLACEHOLDER, fullPackageImportPath)
.replaceAll(MODULE_NAME_PLACEHOLDER, fields.moduleName)
.replaceAll(MODULE_NAME_ALL_CAPS_PLACEHOLDER, moduleNameAllCaps)
.replaceAll(SIDEBAR_LABEL_PLACEHOLDER, sidebarLabel)
.replaceAll(MODULE_NAME_PLACEHOLDER, fields.className)
.replaceAll(PACKAGE_NAME_PLACEHOLDER, extraFields.packageName)
.replaceAll(FULL_IMPORT_PATH_PLACEHOLDER, extraFields.fullImportPath)
.replaceAll(ENV_VAR_NAME_PLACEHOLDER, extraFields.envVarName)
.replaceAll(API_REF_MODULE_PLACEHOLDER, apiRefModuleUrl)
.replaceAll(API_REF_PACKAGE_PLACEHOLDER, apiRefPackageUrl)
.replaceAll(PYTHON_DOC_URL_PLACEHOLDER, pyDocUrl)
.replaceAll(
TOOL_CALLING_PLACEHOLDER,
extraFields?.toolCalling ? "βœ…" : "❌"
Expand All @@ -237,10 +219,8 @@ export async function fillChatIntegrationDocTemplate(fields: {
)
.replace(PY_SUPPORT_PLACEHOLDER, extraFields?.pySupport ? "βœ…" : "❌");

const docPath = path.join(
INTEGRATIONS_DOCS_PATH,
`${packageNameShortSnakeCase}.ipynb`
);
const docFileName = extraFields.fullImportPath.split("/").pop();
const docPath = path.join(INTEGRATIONS_DOCS_PATH, `${docFileName}.ipynb`);
await fs.promises.writeFile(docPath, docTemplate);
const prettyDocPath = docPath.split("docs/core_docs/")[1];

Expand Down
Loading

0 comments on commit 589354f

Please sign in to comment.