From 7ecf38f4facfa36bc630f91fc646adc04796d855 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Thu, 5 Dec 2024 16:32:47 -0800 Subject: [PATCH] cli: create specific files from template (#28556) --- libs/cli/Makefile | 4 +- .../langchain_cli/namespaces/integration.py | 132 +++++++++++------- 2 files changed, 82 insertions(+), 54 deletions(-) diff --git a/libs/cli/Makefile b/libs/cli/Makefile index ac855bd4dc58a..1560bfa65505d 100644 --- a/libs/cli/Makefile +++ b/libs/cli/Makefile @@ -38,10 +38,12 @@ _e2e_test: $(PYTHON) -m pip install --upgrade poetry && \ $(PYTHON) -m pip install -e .. && \ $(PYTHON) -m langchain_cli.cli integration new --name parrot-link --name-class ParrotLink && \ + $(PYTHON) -m langchain_cli.cli integration new --name parrot-link --name-class ParrotLinkB --src=integration_template/chat_models.py --dst=langchain-parrot-link/langchain_parrot_link/chat_models_b.py && \ + $(PYTHON) -m langchain_cli.cli integration create-doc --name parrot-link --name-class ParrotLinkB --component-type ChatModel --destination-dir langchain-parrot-link/docs && \ cd langchain-parrot-link && \ poetry install --with lint,typing,test && \ poetry run pip install -e ../../../standard-tests && \ make format lint tests && \ poetry install --with test_integration && \ rm tests/integration_tests/test_vectorstores.py && \ - make integration_test + make integration_test diff --git a/libs/cli/langchain_cli/namespaces/integration.py b/libs/cli/langchain_cli/namespaces/integration.py index eca9804358561..fbacf1274f177 100644 --- a/libs/cli/langchain_cli/namespaces/integration.py +++ b/libs/cli/langchain_cli/namespaces/integration.py @@ -69,6 +69,21 @@ def new( " This is used to name classes like `MyIntegrationVectorStore`" ), ] = None, + src: Annotated[ + Optional[list[str]], + typer.Option( + help="The name of the single template file to copy." + " e.g. `--src integration_template/chat_models.py " + "--dst my_integration/chat_models.py`. Can be used multiple times.", + ), + ] = None, + dst: Annotated[ + Optional[list[str]], + typer.Option( + help="The relative path to the integration package to place the new file in" + ". e.g. `my-integration/my_integration.py`", + ), + ] = None, ): """ Creates a new integration package. @@ -93,27 +108,66 @@ def new( "Name of integration in PascalCase", default=replacements["__ModuleName__"] ) + project_template_dir = Path(__file__).parents[1] / "integration_template" destination_dir = Path.cwd() / replacements["__package_name__"] - if destination_dir.exists(): - typer.echo(f"Folder {destination_dir} exists.") - raise typer.Exit(code=1) + if not src and not dst: + if destination_dir.exists(): + typer.echo(f"Folder {destination_dir} exists.") + raise typer.Exit(code=1) - # copy over template from ../integration_template - project_template_dir = Path(__file__).parents[1] / "integration_template" - shutil.copytree(project_template_dir, destination_dir, dirs_exist_ok=False) + # copy over template from ../integration_template + shutil.copytree(project_template_dir, destination_dir, dirs_exist_ok=False) - # folder movement - package_dir = destination_dir / replacements["__module_name__"] - shutil.move(destination_dir / "integration_template", package_dir) + # folder movement + package_dir = destination_dir / replacements["__module_name__"] + shutil.move(destination_dir / "integration_template", package_dir) - # replacements in files - replace_glob(destination_dir, "**/*", cast(Dict[str, str], replacements)) + # replacements in files + replace_glob(destination_dir, "**/*", cast(Dict[str, str], replacements)) - # poetry install - subprocess.run( - ["poetry", "install", "--with", "lint,test,typing,test_integration"], - cwd=destination_dir, - ) + # poetry install + subprocess.run( + ["poetry", "install", "--with", "lint,test,typing,test_integration"], + cwd=destination_dir, + ) + else: + # confirm src and dst are the same length + if not src: + typer.echo("Cannot provide --dst without --src.") + raise typer.Exit(code=1) + src_paths = [project_template_dir / p for p in src] + if dst and len(src) != len(dst): + typer.echo("Number of --src and --dst arguments must match.") + raise typer.Exit(code=1) + if not dst: + # assume we're in a package dir, copy to equivalent path + dst_paths = [destination_dir / p for p in src] + else: + dst_paths = [Path.cwd() / p for p in dst] + dst_paths = [ + p / f"{replacements['__package_name_short_snake__']}.ipynb" + if not p.suffix + else p + for p in dst_paths + ] + + # confirm no duplicate dst_paths + if len(dst_paths) != len(set(dst_paths)): + typer.echo( + "Duplicate destination paths provided or computed - please " + "specify them explicitly with --dst." + ) + raise typer.Exit(code=1) + + # confirm no files exist at dst_paths + for dst_path in dst_paths: + if dst_path.exists(): + typer.echo(f"File {dst_path} exists.") + raise typer.Exit(code=1) + + for src_path, dst_path in zip(src_paths, dst_paths): + shutil.copy(src_path, dst_path) + replace_file(dst_path, cast(Dict[str, str], replacements)) TEMPLATE_MAP: dict[str, str] = { @@ -176,43 +230,15 @@ def create_doc( """ Creates a new integration doc. """ - try: - replacements = _process_name(name, community=component_type == "Tool") - except ValueError as e: - typer.echo(e) - raise typer.Exit(code=1) - - if name_class: - if not re.match(r"^[A-Z][a-zA-Z0-9]*$", name_class): - typer.echo( - "Name should only contain letters (a-z, A-Z), numbers, and underscores" - ", and start with a capital letter." - ) - raise typer.Exit(code=1) - replacements["__ModuleName__"] = name_class - else: - replacements["__ModuleName__"] = typer.prompt( - ( - "The PascalCase name of the integration (e.g. `OpenAI`, `VertexAI`). " - "Do not include a 'Chat', 'VectorStore', etc. prefix/suffix." - ), - default=replacements["__ModuleName__"], - ) - destination_path = ( - Path.cwd() - / destination_dir - / (replacements["__package_name_short_snake__"] + ".ipynb") - ) - - # copy over template from ../integration_template - template_dir = Path(__file__).parents[1] / "integration_template" / "docs" - if component_type in TEMPLATE_MAP: - docs_template = template_dir / TEMPLATE_MAP[component_type] - else: - raise ValueError( + if component_type not in TEMPLATE_MAP: + typer.echo( f"Unrecognized {component_type=}. Expected one of {_component_types_str}." ) - shutil.copy(docs_template, destination_path) + raise typer.Exit(code=1) - # replacements in file - replace_file(destination_path, cast(Dict[str, str], replacements)) + new( + name=name, + name_class=name_class, + src=[f"docs/{TEMPLATE_MAP[component_type]}"], + dst=[destination_dir], + )