From ab446dfd91889552a55e158e62fc94a4c2e260bb Mon Sep 17 00:00:00 2001 From: William FH <13333726+hinthornw@users.noreply.github.com> Date: Tue, 10 Sep 2024 10:49:04 -0700 Subject: [PATCH] Pydocs (#991) --- .readthedocs.yml | 35 ++ python/Makefile | 2 +- python/docs/.gitignore | 4 + python/docs/.python-version | 1 + python/docs/Makefile | 34 ++ python/docs/_extensions/gallery_directive.py | 144 +++++++ python/docs/_static/css/custom.css | 411 +++++++++++++++++++ python/docs/_static/img/brand/favicon.png | Bin 0 -> 777 bytes python/docs/_static/wordmark-api-dark.svg | 11 + python/docs/_static/wordmark-api.svg | 11 + python/docs/conf.py | 261 ++++++++++++ python/docs/create_api_rst.py | 372 +++++++++++++++++ python/docs/make.bat | 35 ++ python/docs/requirements.txt | 12 + python/docs/scripts/custom_formatter.py | 41 ++ python/docs/templates/COPYRIGHT.txt | 27 ++ python/docs/templates/langsmith_docs.html | 12 + python/docs/templates/redirects.html | 16 + python/langsmith/anonymizer.py | 15 + python/langsmith/async_client.py | 56 ++- python/langsmith/client.py | 284 ++----------- python/langsmith/evaluation/__init__.py | 1 - python/langsmith/evaluation/_runner.py | 4 +- python/langsmith/run_helpers.py | 154 +++---- python/langsmith/run_trees.py | 3 +- python/langsmith/schemas.py | 124 +++--- python/langsmith/utils.py | 2 +- python/pyproject.toml | 20 +- python/tests/evaluation/__init__.py | 31 ++ 29 files changed, 1703 insertions(+), 420 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 python/docs/.gitignore create mode 100644 python/docs/.python-version create mode 100644 python/docs/Makefile create mode 100644 python/docs/_extensions/gallery_directive.py create mode 100644 python/docs/_static/css/custom.css create mode 100644 python/docs/_static/img/brand/favicon.png create mode 100644 python/docs/_static/wordmark-api-dark.svg create mode 100644 python/docs/_static/wordmark-api.svg create mode 100644 python/docs/conf.py create mode 100644 python/docs/create_api_rst.py create mode 100644 python/docs/make.bat create mode 100644 python/docs/requirements.txt create mode 100644 python/docs/scripts/custom_formatter.py create mode 100644 python/docs/templates/COPYRIGHT.txt create mode 100644 python/docs/templates/langsmith_docs.html create mode 100644 python/docs/templates/redirects.html diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000..98b654db8 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,35 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +formats: + - pdf + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + commands: + - mkdir -p $READTHEDOCS_OUTPUT + - echo "Building docs" + - pip install -U uv + - uv venv + - . .venv/bin/activate + - uv pip install -r python/docs/requirements.txt + - . .venv/bin/activate && cd python/docs && make clobber generate-api-rst html && cd ../.. + - cp -r python/docs/_build/html $READTHEDOCS_OUTPUT +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: python/docs/conf.py + +# If using Sphinx, optionally build your docs in additional formats such as PDF +# formats: +# - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: python/docs/requirements.txt diff --git a/python/Makefile b/python/Makefile index d06830bf9..e40f8944b 100644 --- a/python/Makefile +++ b/python/Makefile @@ -20,7 +20,7 @@ evals: lint: poetry run ruff check . - poetry run mypy . + poetry run mypy langsmith poetry run black . --check format: diff --git a/python/docs/.gitignore b/python/docs/.gitignore new file mode 100644 index 000000000..ac2deeb04 --- /dev/null +++ b/python/docs/.gitignore @@ -0,0 +1,4 @@ +_build/ +langsmith/ +index.rst + diff --git a/python/docs/.python-version b/python/docs/.python-version new file mode 100644 index 000000000..2c0733315 --- /dev/null +++ b/python/docs/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/python/docs/Makefile b/python/docs/Makefile new file mode 100644 index 000000000..7ac449a0d --- /dev/null +++ b/python/docs/Makefile @@ -0,0 +1,34 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= -j auto +SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD ?= sphinx-autobuild +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile +# Generate API reference RST files +generate-api-rst: + python ./create_api_rst.py + +# Combined target to generate API RST and build HTML +api-docs: generate-api-rst build-html + +.PHONY: generate-api-rst build-html api-docs + +clobber: clean + rm -rf langsmith + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @echo "SOURCEDIR: $(SOURCEDIR)" + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + diff --git a/python/docs/_extensions/gallery_directive.py b/python/docs/_extensions/gallery_directive.py new file mode 100644 index 000000000..80642c545 --- /dev/null +++ b/python/docs/_extensions/gallery_directive.py @@ -0,0 +1,144 @@ +"""A directive to generate a gallery of images from structured data. + +Generating a gallery of images that are all the same size is a common +pattern in documentation, and this can be cumbersome if the gallery is +generated programmatically. This directive wraps this particular use-case +in a helper-directive to generate it with a single YAML configuration file. + +It currently exists for maintainers of the pydata-sphinx-theme, +but might be abstracted into a standalone package if it proves useful. +""" + +from pathlib import Path +from typing import Any, ClassVar, Dict, List + +from docutils import nodes +from docutils.parsers.rst import directives +from sphinx.application import Sphinx +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective +from yaml import safe_load + +logger = logging.getLogger(__name__) + + +TEMPLATE_GRID = """ +`````{{grid}} {columns} +{options} + +{content} + +````` +""" + +GRID_CARD = """ +````{{grid-item-card}} {title} +{options} + +{content} +```` +""" + + +class GalleryGridDirective(SphinxDirective): + """A directive to show a gallery of images and links in a Bootstrap grid. + + The grid can be generated from a YAML file that contains a list of items, or + from the content of the directive (also formatted in YAML). Use the parameter + "class-card" to add an additional CSS class to all cards. When specifying the grid + items, you can use all parameters from "grid-item-card" directive to customize + individual cards + ["image", "header", "content", "title"]. + + Danger: + This directive can only be used in the context of a Myst documentation page as + the templates use Markdown flavored formatting. + """ + + name = "gallery-grid" + has_content = True + required_arguments = 0 + optional_arguments = 1 + final_argument_whitespace = True + option_spec: ClassVar[dict[str, Any]] = { + # A class to be added to the resulting container + "grid-columns": directives.unchanged, + "class-container": directives.unchanged, + "class-card": directives.unchanged, + } + + def run(self) -> List[nodes.Node]: + """Create the gallery grid.""" + if self.arguments: + # If an argument is given, assume it's a path to a YAML file + # Parse it and load it into the directive content + path_data_rel = Path(self.arguments[0]) + path_doc, _ = self.get_source_info() + path_doc = Path(path_doc).parent + path_data = (path_doc / path_data_rel).resolve() + if not path_data.exists(): + logger.info(f"Could not find grid data at {path_data}.") + nodes.text("No grid data found at {path_data}.") + return + yaml_string = path_data.read_text() + else: + yaml_string = "\n".join(self.content) + + # Use all the element with an img-bottom key as sites to show + # and generate a card item for each of them + grid_items = [] + for item in safe_load(yaml_string): + # remove parameters that are not needed for the card options + title = item.pop("title", "") + + # build the content of the card using some extra parameters + header = f"{item.pop('header')} \n^^^ \n" if "header" in item else "" + image = f"![image]({item.pop('image')}) \n" if "image" in item else "" + content = f"{item.pop('content')} \n" if "content" in item else "" + + # optional parameter that influence all cards + if "class-card" in self.options: + item["class-card"] = self.options["class-card"] + + loc_options_str = "\n".join(f":{k}: {v}" for k, v in item.items()) + " \n" + + card = GRID_CARD.format( + options=loc_options_str, content=header + image + content, title=title + ) + grid_items.append(card) + + # Parse the template with Sphinx Design to create an output container + # Prep the options for the template grid + class_ = "gallery-directive" + f' {self.options.get("class-container", "")}' + options = {"gutter": 2, "class-container": class_} + options_str = "\n".join(f":{k}: {v}" for k, v in options.items()) + + # Create the directive string for the grid + grid_directive = TEMPLATE_GRID.format( + columns=self.options.get("grid-columns", "1 2 3 4"), + options=options_str, + content="\n".join(grid_items), + ) + + # Parse content as a directive so Sphinx Design processes it + container = nodes.container() + self.state.nested_parse([grid_directive], 0, container) + + # Sphinx Design outputs a container too, so just use that + return [container.children[0]] + + +def setup(app: Sphinx) -> Dict[str, Any]: + """Add custom configuration to sphinx app. + + Args: + app: the Sphinx application + + Returns: + the 2 parallel parameters set to ``True``. + """ + app.add_directive("gallery-grid", GalleryGridDirective) + + return { + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/python/docs/_static/css/custom.css b/python/docs/_static/css/custom.css new file mode 100644 index 000000000..87195de8f --- /dev/null +++ b/python/docs/_static/css/custom.css @@ -0,0 +1,411 @@ +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;700&display=swap'); + +/******************************************************************************* +* master color map. Only the colors that actually differ between light and dark +* themes are specified separately. +* +* To see the full list of colors see https://www.figma.com/file/rUrrHGhUBBIAAjQ82x6pz9/PyData-Design-system---proposal-for-implementation-(2)?node-id=1234%3A765&t=ifcFT1JtnrSshGfi-1 +*/ +/** +* Function to get items from nested maps +*/ +/* Assign base colors for the PyData theme */ +:root { + --pst-teal-50: #f4fbfc; + --pst-teal-100: #e9f6f8; + --pst-teal-200: #d0ecf1; + --pst-teal-300: #abdde6; + --pst-teal-400: #3fb1c5; + --pst-teal-500: #0a7d91; + --pst-teal-600: #085d6c; + --pst-teal-700: #064752; + --pst-teal-800: #042c33; + --pst-teal-900: #021b1f; + --pst-violet-50: #f4eefb; + --pst-violet-100: #e0c7ff; + --pst-violet-200: #d5b4fd; + --pst-violet-300: #b780ff; + --pst-violet-400: #9c5ffd; + --pst-violet-500: #8045e5; + --pst-violet-600: #6432bd; + --pst-violet-700: #4b258f; + --pst-violet-800: #341a61; + --pst-violet-900: #1e0e39; + --pst-gray-50: #f9f9fa; + --pst-gray-100: #f3f4f5; + --pst-gray-200: #e5e7ea; + --pst-gray-300: #d1d5da; + --pst-gray-400: #9ca4af; + --pst-gray-500: #677384; + --pst-gray-600: #48566b; + --pst-gray-700: #29313d; + --pst-gray-800: #222832; + --pst-gray-900: #14181e; + --pst-pink-50: #fcf8fd; + --pst-pink-100: #fcf0fa; + --pst-pink-200: #f8dff5; + --pst-pink-300: #f3c7ee; + --pst-pink-400: #e47fd7; + --pst-pink-500: #c132af; + --pst-pink-600: #912583; + --pst-pink-700: #6e1c64; + --pst-pink-800: #46123f; + --pst-pink-900: #2b0b27; + --pst-foundation-white: #ffffff; + --pst-foundation-black: #14181e; + --pst-green-10: #f1fdfd; + --pst-green-50: #E0F7F6; + --pst-green-100: #B3E8E6; + --pst-green-200: #80D6D3; + --pst-green-300: #4DC4C0; + --pst-green-400: #4FB2AD; + --pst-green-500: #287977; + --pst-green-600: #246161; + --pst-green-700: #204F4F; + --pst-green-800: #1C3C3C; + --pst-green-900: #0D2427; + --pst-lilac-50: #f4eefb; + --pst-lilac-100: #DAD6FE; + --pst-lilac-200: #BCB2FD; + --pst-lilac-300: #9F8BFA; + --pst-lilac-400: #7F5CF6; + --pst-lilac-500: #6F3AED; + --pst-lilac-600: #6028D9; + --pst-lilac-700: #5021B6; + --pst-lilac-800: #431D95; + --pst-lilac-900: #1e0e39; + --pst-header-height: 2.5rem; +} + +html { + --pst-font-family-base: 'Inter'; + --pst-font-family-heading: 'Inter Tight', sans-serif; +} + +/******************************************************************************* +* write the color rules for each theme (light/dark) +*/ +/* NOTE: + * Mixins enable us to reuse the same definitions for the different modes + * https://sass-lang.com/documentation/at-rules/mixin + * something inserts a variable into a CSS selector or property name + * https://sass-lang.com/documentation/interpolation + */ +/* Defaults to light mode if data-theme is not set */ +html:not([data-theme]) { + --pst-color-primary: #287977; + --pst-color-primary-bg: #80D6D3; + --pst-color-secondary: #6F3AED; + --pst-color-secondary-bg: #DAD6FE; + --pst-color-accent: #c132af; + --pst-color-accent-bg: #f8dff5; + --pst-color-info: #276be9; + --pst-color-info-bg: #dce7fc; + --pst-color-warning: #f66a0a; + --pst-color-warning-bg: #f8e3d0; + --pst-color-success: #00843f; + --pst-color-success-bg: #d6ece1; + --pst-color-attention: var(--pst-color-warning); + --pst-color-attention-bg: var(--pst-color-warning-bg); + --pst-color-danger: #d72d47; + --pst-color-danger-bg: #f9e1e4; + --pst-color-text-base: #222832; + --pst-color-text-muted: #48566b; + --pst-color-heading-color: #ffffff; + --pst-color-shadow: rgba(0, 0, 0, 0.1); + --pst-color-border: #d1d5da; + --pst-color-border-muted: rgba(23, 23, 26, 0.2); + --pst-color-inline-code: #912583; + --pst-color-inline-code-links: #246161; + --pst-color-target: #f3cf95; + --pst-color-background: #ffffff; + --pst-color-on-background: #F4F9F8; + --pst-color-surface: #F4F9F8; + --pst-color-on-surface: #222832; +} +html:not([data-theme]) { + --pst-color-link: var(--pst-color-primary); + --pst-color-link-hover: var(--pst-color-secondary); +} +html:not([data-theme]) .only-dark, +html:not([data-theme]) .only-dark ~ figcaption { + display: none !important; +} + +/* NOTE: @each {...} is like a for-loop + * https://sass-lang.com/documentation/at-rules/control/each + */ +html[data-theme=light] { + --pst-color-primary: #287977; + --pst-color-primary-bg: #80D6D3; + --pst-color-secondary: #6F3AED; + --pst-color-secondary-bg: #DAD6FE; + --pst-color-accent: #c132af; + --pst-color-accent-bg: #f8dff5; + --pst-color-info: #276be9; + --pst-color-info-bg: #dce7fc; + --pst-color-warning: #f66a0a; + --pst-color-warning-bg: #f8e3d0; + --pst-color-success: #00843f; + --pst-color-success-bg: #d6ece1; + --pst-color-attention: var(--pst-color-warning); + --pst-color-attention-bg: var(--pst-color-warning-bg); + --pst-color-danger: #d72d47; + --pst-color-danger-bg: #f9e1e4; + --pst-color-text-base: #222832; + --pst-color-text-muted: #48566b; + --pst-color-heading-color: #ffffff; + --pst-color-shadow: rgba(0, 0, 0, 0.1); + --pst-color-border: #d1d5da; + --pst-color-border-muted: rgba(23, 23, 26, 0.2); + --pst-color-inline-code: #912583; + --pst-color-inline-code-links: #246161; + --pst-color-target: #f3cf95; + --pst-color-background: #ffffff; + --pst-color-on-background: #F4F9F8; + --pst-color-surface: #F4F9F8; + --pst-color-on-surface: #222832; + color-scheme: light; +} +html[data-theme=light] { + --pst-color-link: var(--pst-color-primary); + --pst-color-link-hover: var(--pst-color-secondary); +} +html[data-theme=light] .only-dark, +html[data-theme=light] .only-dark ~ figcaption { + display: none !important; +} + +html[data-theme=dark] { + --pst-color-primary: #4FB2AD; + --pst-color-primary-bg: #1C3C3C; + --pst-color-secondary: #7F5CF6; + --pst-color-secondary-bg: #431D95; + --pst-color-accent: #e47fd7; + --pst-color-accent-bg: #46123f; + --pst-color-info: #79a3f2; + --pst-color-info-bg: #06245d; + --pst-color-warning: #ff9245; + --pst-color-warning-bg: #652a02; + --pst-color-success: #5fb488; + --pst-color-success-bg: #002f17; + --pst-color-attention: var(--pst-color-warning); + --pst-color-attention-bg: var(--pst-color-warning-bg); + --pst-color-danger: #e78894; + --pst-color-danger-bg: #4e111b; + --pst-color-text-base: #ced6dd; + --pst-color-text-muted: #9ca4af; + --pst-color-heading-color: #14181e; + --pst-color-shadow: rgba(0, 0, 0, 0.2); + --pst-color-border: #48566b; + --pst-color-border-muted: #29313d; + --pst-color-inline-code: #f3c7ee; + --pst-color-inline-code-links: #4FB2AD; + --pst-color-target: #675c04; + --pst-color-background: #14181e; + --pst-color-on-background: #222832; + --pst-color-surface: #29313d; + --pst-color-on-surface: #f3f4f5; + /* Adjust images in dark mode (unless they have class .only-dark or + * .dark-light, in which case assume they're already optimized for dark + * mode). + */ + /* Give images a light background in dark mode in case they have + * transparency and black text (unless they have class .only-dark or .dark-light, in + * which case assume they're already optimized for dark mode). + */ + color-scheme: dark; +} +html[data-theme=dark] { + --pst-color-link: var(--pst-color-primary); + --pst-color-link-hover: var(--pst-color-secondary); +} +html[data-theme=dark] .only-light, +html[data-theme=dark] .only-light ~ figcaption { + display: none !important; +} +html[data-theme=dark] img:not(.only-dark):not(.dark-light) { + filter: brightness(0.8) contrast(1.2); +} +html[data-theme=dark] .bd-content img:not(.only-dark):not(.dark-light) { + background: rgb(255, 255, 255); + border-radius: 0.25rem; +} +html[data-theme=dark] .MathJax_SVG * { + fill: var(--pst-color-text-base); +} + +.pst-color-primary { + color: var(--pst-color-primary); +} + +.pst-color-secondary { + color: var(--pst-color-secondary); +} + +.pst-color-accent { + color: var(--pst-color-accent); +} + +.pst-color-info { + color: var(--pst-color-info); +} + +.pst-color-warning { + color: var(--pst-color-warning); +} + +.pst-color-success { + color: var(--pst-color-success); +} + +.pst-color-attention { + color: var(--pst-color-attention); +} + +.pst-color-danger { + color: var(--pst-color-danger); +} + +.pst-color-text-base { + color: var(--pst-color-text-base); +} + +.pst-color-text-muted { + color: var(--pst-color-text-muted); +} + +.pst-color-heading-color { + color: var(--pst-color-heading-color); +} + +.pst-color-shadow { + color: var(--pst-color-shadow); +} + +.pst-color-border { + color: var(--pst-color-border); +} + +.pst-color-border-muted { + color: var(--pst-color-border-muted); +} + +.pst-color-inline-code { + color: var(--pst-color-inline-code); +} + +.pst-color-inline-code-links { + color: var(--pst-color-inline-code-links); +} + +.pst-color-target { + color: var(--pst-color-target); +} + +.pst-color-background { + color: var(--pst-color-background); +} + +.pst-color-on-background { + color: var(--pst-color-on-background); +} + +.pst-color-surface { + color: var(--pst-color-surface); +} + +.pst-color-on-surface { + color: var(--pst-color-on-surface); +} + + + +/* Adjust the height of the navbar */ +.bd-header .bd-header__inner{ + height: 52px; /* Adjust this value as needed */ +} + +.navbar-nav > li > a { + line-height: 52px; /* Vertically center the navbar links */ +} + +/* Make sure the navbar items align properly */ +.navbar-nav { + display: flex; +} + + +.bd-header .navbar-header-items__start{ + margin-left: 0rem +} + +.bd-header button.primary-toggle { + margin-right: 0rem; +} + +.bd-header ul.navbar-nav .dropdown .dropdown-menu { + overflow-y: auto; /* Enable vertical scrolling */ + max-height: 80vh +} + +.bd-sidebar-primary { + width: 22%; /* Adjust this value to your preference */ + line-height: 1.4; +} + +.bd-sidebar-secondary { + line-height: 1.4; +} + +.toc-entry a.nav-link, .toc-entry a>code { + background-color: transparent; + border-color: transparent; +} + +.bd-sidebar-primary code{ + background-color: transparent; + border-color: transparent; +} + + +.toctree-wrapper li[class^=toctree-l1]>a { + font-size: 1.3em +} + +.toctree-wrapper li[class^=toctree-l1] { + margin-bottom: 2em; +} + +.toctree-wrapper li[class^=toctree-l]>ul { + margin-top: 0.5em; + font-size: 0.9em; +} + +*, :after, :before { + font-style: normal; +} + +div.deprecated { + margin-top: 0.5em; + margin-bottom: 2em; +} + +.admonition-beta.admonition, div.admonition-beta.admonition { + border-color: var(--pst-color-warning); + margin-top:0.5em; + margin-bottom: 2em; +} + +.admonition-beta>.admonition-title, div.admonition-beta>.admonition-title { + background-color: var(--pst-color-warning-bg); +} + +dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple) dd { + margin-left: 1rem; +} + +p { + font-size: 0.9rem; + margin-bottom: 0.5rem; +} \ No newline at end of file diff --git a/python/docs/_static/img/brand/favicon.png b/python/docs/_static/img/brand/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..e0335bcb61014e58e40b5028c2b7329d2ceed613 GIT binary patch literal 777 zcmV+k1NQuhP)NE^{Z&zv?I;d@nTi;kP-#MuQ6f+xuyHw)Y}>3T1j)JCCFkTN z(6HZ$z*!&vUJat3KOyq^6`rvnf@E{K6r=OTr7LJ_If}JJIPw&O!*Y&)gT-zSY90Mu zew;aWQnlrWVRUo{F#kIP&%p*P2lJT>9>08n=^x)w$J)24tPD=aUewlD@#g(I-CUgr zsodVy)2dD7lOH`$Ga!2@%%oB{-s!_n*8dM|erK>(z1G%pB&Vf10^Zhk-9<uFg>g5f4KZ{CVvM~A#G16A zWk141sU=*yNRR@?6O$MY4hpxjQOLS&IKwg{rRF0sJ`qE8)mFY!CZ-ek^Y1UOPW6^5 z4BQDJ`ehvVpN#U%4>x&)>kP=Zc$_S94kA*nkpz`(Br?WGn0b$zn_SpazZ=Qatf~)M zG{!P-Jm7@6T&yM0VI({Wv%OqP&{0g=*od5yknr|nlxIX`*NU$YP|tfp*STgy5LUp5 zqTCHvx=^7gJrR`%ln9t4_JW8af!>f!?DHc1z&c|w@vHv + + + + + + + + + + diff --git a/python/docs/_static/wordmark-api.svg b/python/docs/_static/wordmark-api.svg new file mode 100644 index 000000000..a9f8f59db --- /dev/null +++ b/python/docs/_static/wordmark-api.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/python/docs/conf.py b/python/docs/conf.py new file mode 100644 index 000000000..dae18242c --- /dev/null +++ b/python/docs/conf.py @@ -0,0 +1,261 @@ +"""Configuration file for the Sphinx documentation builder.""" + +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +import os +import sys +from pathlib import Path + +import toml +from docutils import nodes +from docutils.parsers.rst.directives.admonitions import BaseAdmonition +from docutils.statemachine import StringList +from sphinx.util.docutils import SphinxDirective + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +_DIR = Path(__file__).parent.absolute() +sys.path.insert(0, os.path.abspath(".")) +sys.path.insert(0, os.path.abspath("../python")) + +with (_DIR.parent / "pyproject.toml").open("r") as f: + data = toml.load(f) + + +class ExampleLinksDirective(SphinxDirective): + """Directive to generate a list of links to examples. + + We have a script that extracts links to API reference docs + from our notebook examples. This directive uses that information + to backlink to the examples from the API reference docs. + """ + + has_content = False + required_arguments = 1 + + def run(self): + """Run the directive. + + Called any time :example_links:`ClassName` is used + in the template *.rst files. + """ + class_or_func_name = self.arguments[0] + links = {} + list_node = nodes.bullet_list() + for doc_name, link in sorted(links.items()): + item_node = nodes.list_item() + para_node = nodes.paragraph() + link_node = nodes.reference() + link_node["refuri"] = link + link_node.append(nodes.Text(doc_name)) + para_node.append(link_node) + item_node.append(para_node) + list_node.append(item_node) + if list_node.children: + title_node = nodes.rubric() + title_node.append(nodes.Text(f"Examples using {class_or_func_name}")) + return [title_node, list_node] + return [list_node] + + +class Beta(BaseAdmonition): + required_arguments = 0 + node_class = nodes.admonition + + def run(self): + self.content = self.content or StringList( + [ + ( + "This feature is in beta. It is actively being worked on, so the " + "API may change." + ) + ] + ) + self.arguments = self.arguments or ["Beta"] + return super().run() + + +def setup(app): + app.add_directive("example_links", ExampleLinksDirective) + app.add_directive("beta", Beta) + + +# -- Project information ----------------------------------------------------- + +project = "🦜️🛠️ LangSmith" +copyright = "2024, LangChain Inc" +author = "LangChain, Inc" + +html_favicon = "_static/img/brand/favicon.png" +html_last_updated_fmt = "%b %d, %Y" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autodoc.typehints", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinxcontrib.autodoc_pydantic", + "IPython.sphinxext.ipython_console_highlighting", + "myst_parser", + "_extensions.gallery_directive", + "sphinx_design", + "sphinx_copybutton", +] +source_suffix = [".rst", ".md"] + +# some autodoc pydantic options are repeated in the actual template. +# potentially user error, but there may be bugs in the sphinx extension +# with options not being passed through correctly (from either the location in the code) +autodoc_pydantic_model_show_json = False +autodoc_pydantic_field_list_validators = False +autodoc_pydantic_config_members = False +autodoc_pydantic_model_show_config_summary = False +autodoc_pydantic_model_show_validator_members = False +autodoc_pydantic_model_show_validator_summary = False +autodoc_pydantic_model_signature_prefix = "class" +autodoc_pydantic_field_signature_prefix = "param" +autodoc_member_order = "groupwise" +autoclass_content = "both" +autodoc_typehints_format = "short" +autodoc_typehints = "both" + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# The theme to use for HTML and HTML Help pages. +html_theme = "pydata_sphinx_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + # # -- General configuration ------------------------------------------------ + "sidebar_includehidden": True, + "use_edit_page_button": False, + # # "analytics": { + # # "plausible_analytics_domain": "scikit-learn.org", + # # "plausible_analytics_url": "https://views.scientific-python.org/js/script.js", + # # }, + # # If "prev-next" is included in article_footer_items, then setting show_prev_next + # # to True would repeat prev and next links. See + # # https://github.com/pydata/pydata-sphinx-theme/blob/b731dc230bc26a3d1d1bb039c56c977a9b3d25d8/src/pydata_sphinx_theme/theme/pydata_sphinx_theme/layout.html#L118-L129 + "show_prev_next": False, + "search_bar_text": "Search", + "navigation_with_keys": True, + "collapse_navigation": True, + "navigation_depth": 3, + "show_nav_level": 1, + "show_toc_level": 3, + "navbar_align": "left", + "header_links_before_dropdown": 5, + "header_dropdown_text": "Modules", + "logo": { + "image_light": "_static/wordmark-api.svg", + "image_dark": "_static/wordmark-api-dark.svg", + }, + "surface_warnings": True, + # # -- Template placement in theme layouts ---------------------------------- + "navbar_start": ["navbar-logo"], + # # Note that the alignment of navbar_center is controlled by navbar_align + "navbar_center": ["navbar-nav"], + "navbar_end": ["langsmith_docs", "theme-switcher", "navbar-icon-links"], + # # navbar_persistent is persistent right (even when on mobiles) + "navbar_persistent": ["search-field"], + "article_header_start": ["breadcrumbs"], + "article_header_end": [], + "article_footer_items": [], + "content_footer_items": [], + # # Use html_sidebars that map page patterns to list of sidebar templates + # "primary_sidebar_end": [], + "footer_start": ["copyright"], + "footer_center": [], + "footer_end": [], + # # When specified as a dictionary, the keys should follow glob-style patterns, as in + # # https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-exclude_patterns + # # In particular, "**" specifies the default for all pages + # # Use :html_theme.sidebar_secondary.remove: for file-wide removal + # "secondary_sidebar_items": {"**": ["page-toc", "sourcelink"]}, + # "show_version_warning_banner": True, + # "announcement": None, + "icon_links": [ + { + # Label for this link + "name": "GitHub", + # URL where the link will redirect + "url": "https://github.com/langchain-ai/langsmith-sdk", # required + # Icon class (if "type": "fontawesome"), or path to local image (if "type": "local") + "icon": "fa-brands fa-square-github", + # The type of image to be used (see below for details) + "type": "fontawesome", + }, + { + "name": "X / Twitter", + "url": "https://twitter.com/langchainai", + "icon": "fab fa-twitter-square", + }, + ], + "icon_links_label": "Quick Links", + "external_links": [], +} + + +html_context = { + "display_github": True, # Integrate GitHub + "github_user": "langchain-ai", # Username + "github_repo": "langsmith-sdk", # Repo name + "github_version": "master", # Version + "conf_py_path": "/docs/api_reference", # Path in the checkout to the docs root +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# These paths are either relative to html_static_path +# or fully qualified paths (e.g. https://...) +html_css_files = ["css/custom.css"] +html_use_index = False + +myst_enable_extensions = ["colon_fence"] + +# generate autosummary even if no references +autosummary_generate = True + +html_copy_source = False +html_show_sourcelink = False + +# Set canonical URL from the Read the Docs Domain +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") + +# Tell Jinja2 templates the build is running on Read the Docs +if os.environ.get("READTHEDOCS", "") == "True": + html_context["READTHEDOCS"] = True + +master_doc = "index" diff --git a/python/docs/create_api_rst.py b/python/docs/create_api_rst.py new file mode 100644 index 000000000..3f51da948 --- /dev/null +++ b/python/docs/create_api_rst.py @@ -0,0 +1,372 @@ +"""Script for auto-generating api_reference.rst.""" + +import importlib +import inspect +import logging +import os +import sys +from enum import Enum +from pathlib import Path +from typing import Dict, List, Literal, Sequence, TypedDict, Union + +import toml +from pydantic import BaseModel + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +ROOT_DIR = Path(__file__).parents[1].absolute() +HERE = Path(__file__).parent +sys.path.insert(0, os.path.abspath(".")) +sys.path.insert(0, os.path.abspath("../")) + +PACKAGE_DIR = ROOT_DIR / "langsmith" +ClassKind = Literal["TypedDict", "Regular", "Pydantic", "enum"] + + +class ClassInfo(TypedDict): + name: str + qualified_name: str + kind: ClassKind + is_public: bool + is_deprecated: bool + + +class FunctionInfo(TypedDict): + name: str + qualified_name: str + is_public: bool + is_deprecated: bool + + +class ModuleMembers(TypedDict): + classes_: Sequence[ClassInfo] + functions: Sequence[FunctionInfo] + + +_EXCLUDED_NAMES = { + "close_session", + "convert_prompt_to_anthropic_format", + "convert_prompt_to_openai_format", + "BaseMessageLike", + "TracingQueueItem", + "filter_logs", + "StringEvaluator", + "LLMEvaluator", + "ensure_traceable", + "RunLikeDict", + "RunTypeEnum", + "is_traceable_function", + "is_async", + "get_run_tree_context", + "as_runnable", + "SupportsLangsmithExtra", + "get_tracing_context", +} + +_EXCLUDED_MODULES = {"cli"} + +_INCLUDED_UTILS = { + "ContextThreadPoolExecutor", + "LangSmithAPIError", + "LangSmithAuthError", + "LangSmithConflictError", + "LangSmithConnectionError", + "LangSmithError", + "LangSmithMissingAPIKeyWarning", + "LangSmithNotFoundError", + "LangSmithRateLimitError", + "LangSmithRetry", + "LangSmithUserError", + "LangSmithWarning", +} + + +def _load_module_members(module_path: str, namespace: str) -> ModuleMembers: + classes_: List[ClassInfo] = [] + functions: List[FunctionInfo] = [] + module = importlib.import_module(module_path) + for name, type_ in inspect.getmembers(module): + if "evaluation" in module_path: + print(module_path, name) + if ( + not hasattr(type_, "__module__") + or type_.__module__ != module_path + or name in _EXCLUDED_NAMES + or (module_path.endswith("utils") and name not in _INCLUDED_UTILS) + ): + logger.info(f"Excluding {module_path}.{name}") + continue + + if inspect.isclass(type_): + kind: ClassKind = ( + "TypedDict" + if type(type_).__name__ in ("_TypedDictMeta", "_TypedDictMeta") + else ( + "enum" + if issubclass(type_, Enum) + else "Pydantic" if issubclass(type_, BaseModel) else "Regular" + ) + ) + classes_.append( + ClassInfo( + name=name, + qualified_name=f"{namespace}.{name}", + kind=kind, + is_public=not name.startswith("_"), + is_deprecated=".. deprecated::" in (type_.__doc__ or ""), + ) + ) + elif inspect.isfunction(type_): + functions.append( + FunctionInfo( + name=name, + qualified_name=f"{namespace}.{name}", + is_public=not name.startswith("_"), + is_deprecated=".. deprecated::" in (type_.__doc__ or ""), + ) + ) + + return ModuleMembers(classes_=classes_, functions=functions) + + +def _load_package_modules( + package_directory: Union[str, Path], +) -> Dict[str, ModuleMembers]: + package_path = Path(package_directory) + modules_by_namespace = {} + package_name = package_path.name + + for file_path in package_path.rglob("*.py"): + if file_path.name.startswith("_") or any( + part.startswith("_") for part in file_path.relative_to(package_path).parts + ): + if file_path.name not in { + "_runner.py", + "_arunner.py", + "_testing.py", + "_expect.py", + }: + continue + + namespace = ( + str(file_path.relative_to(package_path)) + .replace(".py", "") + .replace("/", ".") + ) + top_namespace = namespace.split(".")[0] + if top_namespace in _EXCLUDED_MODULES: + logger.info(f"Excluding module {top_namespace}") + continue + + try: + module_members = _load_module_members( + f"{package_name}.{namespace}", namespace + ) + if top_namespace in modules_by_namespace: + existing = modules_by_namespace[top_namespace] + modules_by_namespace[top_namespace] = ModuleMembers( + classes_=existing["classes_"] + module_members["classes_"], + functions=existing["functions"] + module_members["functions"], + ) + else: + modules_by_namespace[top_namespace] = module_members + except ImportError as e: + print(f"Error: Unable to import module '{namespace}' with error: {e}") + + return modules_by_namespace + + +module_order = [ + "client", + "async_client", + "evaluation", + "run_helpers", + "run_trees", + "schemas", + "utils", + "anonymizer", +] + + +def _construct_doc( + package_namespace: str, + members_by_namespace: Dict[str, ModuleMembers], + package_version: str, +) -> List[tuple[str, str]]: + docs = [] + index_doc = f"""\ +:html_theme.sidebar_secondary.remove: + +.. currentmodule:: {package_namespace} + +.. _{package_namespace}: + +{package_namespace.replace('_', '-')}: {package_version} +{'=' * (len(package_namespace) + len(package_version) + 2)} + +.. automodule:: {package_namespace} + :no-members: + :no-inherited-members: + +.. toctree:: + :maxdepth: 2 + +""" + + def _priority(mod: str): + if mod in module_order: + return module_order.index(mod) + print(mod, "not in ", module_order) + return len(module_order) + hash(mod) + + for module in sorted(members_by_namespace, key=lambda x: _priority(x)): + index_doc += f" {module}\n" + module_doc = f"""\ +.. currentmodule:: {package_namespace} + +.. _{package_namespace}_{module}: + +:mod:`{module}` +{'=' * (len(module) + 7)} + +.. automodule:: {package_namespace}.{module} + :no-members: + :no-inherited-members: + +""" + _members = members_by_namespace[module] + classes = [ + el + for el in _members["classes_"] + if el["is_public"] and not el["is_deprecated"] + ] + functions = [ + el + for el in _members["functions"] + if el["is_public"] and not el["is_deprecated"] + ] + deprecated_classes = [ + el for el in _members["classes_"] if el["is_public"] and el["is_deprecated"] + ] + deprecated_functions = [ + el + for el in _members["functions"] + if el["is_public"] and el["is_deprecated"] + ] + + if classes: + module_doc += f"""\ +**Classes** + +.. currentmodule:: {package_namespace} + +.. autosummary:: + :toctree: {module} +""" + for class_ in sorted(classes, key=lambda c: c["qualified_name"]): + template = ( + "typeddict.rst" + if class_["kind"] == "TypedDict" + else ( + "enum.rst" + if class_["kind"] == "enum" + else ( + "pydantic.rst" + if class_["kind"] == "Pydantic" + else "class.rst" + ) + ) + ) + module_doc += f"""\ + :template: {template} + + {class_["qualified_name"]} + +""" + + if functions: + qualnames = "\n ".join(sorted(f["qualified_name"] for f in functions)) + module_doc += f"""**Functions** + +.. currentmodule:: {package_namespace} + +.. autosummary:: + :toctree: {module} + :template: function.rst + + {qualnames} + +""" + + if deprecated_classes: + module_doc += f"""**Deprecated classes** + +.. currentmodule:: {package_namespace} + +.. autosummary:: + :toctree: {module} +""" + for class_ in sorted(deprecated_classes, key=lambda c: c["qualified_name"]): + template = ( + "typeddict.rst" + if class_["kind"] == "TypedDict" + else ( + "enum.rst" + if class_["kind"] == "enum" + else ( + "pydantic.rst" + if class_["kind"] == "Pydantic" + else "class.rst" + ) + ) + ) + module_doc += f""" :template: {template} + + {class_["qualified_name"]} + +""" + + if deprecated_functions: + qualnames = "\n ".join( + sorted(f["qualified_name"] for f in deprecated_functions) + ) + module_doc += f"""**Deprecated functions** + +.. currentmodule:: {package_namespace} + +.. autosummary:: + :toctree: {module} + :template: function.rst + + {qualnames} + +""" + docs.append((f"{module}.rst", module_doc)) + docs.append(("index.rst", index_doc)) + return docs + + +def _get_package_version(package_dir: Path) -> str: + try: + with open(package_dir.parent / "pyproject.toml") as f: + pyproject = toml.load(f) + return pyproject["tool"]["poetry"]["version"] + except FileNotFoundError: + print(f"pyproject.toml not found in {package_dir.parent}. Aborting the build.") + sys.exit(1) + + +def main() -> None: + print("Starting to build API reference files.") + package_members = _load_package_modules(PACKAGE_DIR) + package_version = _get_package_version(PACKAGE_DIR) + rsts = _construct_doc("langsmith", package_members, package_version) + for name, rst in rsts: + with open(HERE / name, "w") as f: + f.write(rst) + print("API reference files built.") + + +if __name__ == "__main__": + main() diff --git a/python/docs/make.bat b/python/docs/make.bat new file mode 100644 index 000000000..922152e96 --- /dev/null +++ b/python/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/python/docs/requirements.txt b/python/docs/requirements.txt new file mode 100644 index 000000000..c93e13128 --- /dev/null +++ b/python/docs/requirements.txt @@ -0,0 +1,12 @@ +autodoc_pydantic>=1,<2 +sphinx<=7 +myst-parser>=3 +sphinx-autobuild>=2024 +pydata-sphinx-theme>=0.15 +toml>=0.10.2 +myst-nb>=1.1.1 +pyyaml +sphinx-design +sphinx-copybutton +beautifulsoup4 +-e python diff --git a/python/docs/scripts/custom_formatter.py b/python/docs/scripts/custom_formatter.py new file mode 100644 index 000000000..ba85484e9 --- /dev/null +++ b/python/docs/scripts/custom_formatter.py @@ -0,0 +1,41 @@ +import sys +from glob import glob +from pathlib import Path + +from bs4 import BeautifulSoup + +CUR_DIR = Path(__file__).parents[1] + + +def process_toc_h3_elements(html_content: str) -> str: + """Update Class.method() TOC headers to just method().""" + # Create a BeautifulSoup object + soup = BeautifulSoup(html_content, "html.parser") + + # Find all
  • elements with class "toc-h3" + toc_h3_elements = soup.find_all("li", class_="toc-h3") + + # Process each element + for element in toc_h3_elements: + element = element.a.code.span + # Get the text content of the element + content = element.get_text() + + # Apply the regex substitution + modified_content = content.split(".")[-1] + + # Update the element's content + element.string = modified_content + + # Return the modified HTML + return str(soup) + + +if __name__ == "__main__": + dir = sys.argv[1] + for fn in glob(str(f"{dir.rstrip('/')}/**/*.html"), recursive=True): + with open(fn) as f: + html = f.read() + processed_html = process_toc_h3_elements(html) + with open(fn, "w") as f: + f.write(processed_html) diff --git a/python/docs/templates/COPYRIGHT.txt b/python/docs/templates/COPYRIGHT.txt new file mode 100644 index 000000000..d4cc36d6b --- /dev/null +++ b/python/docs/templates/COPYRIGHT.txt @@ -0,0 +1,27 @@ +Copyright (c) 2007-2023 The scikit-learn developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/python/docs/templates/langsmith_docs.html b/python/docs/templates/langsmith_docs.html new file mode 100644 index 000000000..7e51aa56e --- /dev/null +++ b/python/docs/templates/langsmith_docs.html @@ -0,0 +1,12 @@ + + + + + +Docs + diff --git a/python/docs/templates/redirects.html b/python/docs/templates/redirects.html new file mode 100644 index 000000000..d76738f5f --- /dev/null +++ b/python/docs/templates/redirects.html @@ -0,0 +1,16 @@ +{% set redirect = pathto(redirects[pagename]) %} + + + + + + + + + + LangSmith Python SDK Reference Documentation. + + +

    You will be automatically redirected to the new location of this page.

    + + diff --git a/python/langsmith/anonymizer.py b/python/langsmith/anonymizer.py index 77e1136f6..02954d460 100644 --- a/python/langsmith/anonymizer.py +++ b/python/langsmith/anonymizer.py @@ -82,6 +82,11 @@ class RuleNodeProcessor(StringNodeProcessor): """String node processor that uses a list of rules to replace sensitive data.""" rules: List[StringNodeRule] + """List of rules to apply for replacing sensitive data. + + Each rule is a StringNodeRule, which contains a regex pattern to match + and an optional replacement string. + """ def __init__(self, rules: List[StringNodeRule]): """Initialize the processor with a list of rules.""" @@ -110,7 +115,17 @@ class CallableNodeProcessor(StringNodeProcessor): """String node processor that uses a callable function to replace sensitive data.""" func: Union[Callable[[str], str], Callable[[str, List[Union[str, int]]], str]] + """The callable function used to replace sensitive data. + + It can be either a function that takes a single string argument and returns a string, + or a function that takes a string and a list of path elements (strings or integers) + and returns a string.""" + accepts_path: bool + """Indicates whether the callable function accepts a path argument. + + If True, the function expects two arguments: the string to be processed and the path to that string. + If False, the function expects only the string to be processed.""" def __init__( self, diff --git a/python/langsmith/async_client.py b/python/langsmith/async_client.py index faa5cf901..8245edbab 100644 --- a/python/langsmith/async_client.py +++ b/python/langsmith/async_client.py @@ -66,7 +66,7 @@ def __init__( ) self._web_url = web_url - async def __aenter__(self) -> "AsyncClient": + async def __aenter__(self) -> AsyncClient: """Enter the async client.""" return self @@ -123,10 +123,8 @@ async def _aget_paginated_list( params["limit"] = params.get("limit", 100) while True: params["offset"] = offset - print(f"path: {path}, params: {params}", flush=True) response = await self._arequest_with_retries("GET", path, params=params) items = response.json() - print(f"items: {items}, response: {response}", flush=True) if not items: break for item in items: @@ -282,62 +280,90 @@ async def list_runs( Examples: -------- + List all runs in a project: + .. code-block:: python - # List all runs in a project project_runs = client.list_runs(project_name="") - # List LLM and Chat runs in the last 24 hours + List LLM and Chat runs in the last 24 hours: + + .. code-block:: python + todays_llm_runs = client.list_runs( project_name="", start_time=datetime.now() - timedelta(days=1), run_type="llm", ) - # List root traces in a project + List root traces in a project: + + .. code-block:: python + root_runs = client.list_runs(project_name="", is_root=1) - # List runs without errors + List runs without errors: + + .. code-block:: python + correct_runs = client.list_runs(project_name="", error=False) - # List runs and only return their inputs/outputs (to speed up the query) + List runs and only return their inputs/outputs (to speed up the query): + + .. code-block:: python + input_output_runs = client.list_runs( project_name="", select=["inputs", "outputs"] ) - # List runs by run ID + List runs by run ID: + + .. code-block:: python + run_ids = [ "a36092d2-4ad5-4fb4-9c0d-0dba9a2ed836", "9398e6be-964f-4aa4-8ae9-ad78cd4b7074", ] selected_runs = client.list_runs(id=run_ids) - # List all "chain" type runs that took more than 10 seconds and had - # `total_tokens` greater than 5000 + List all "chain" type runs that took more than 10 seconds and had + `total_tokens` greater than 5000: + + .. code-block:: python + chain_runs = client.list_runs( project_name="", filter='and(eq(run_type, "chain"), gt(latency, 10), gt(total_tokens, 5000))', ) - # List all runs called "extractor" whose root of the trace was assigned feedback "user_score" score of 1 + List all runs called "extractor" whose root of the trace was assigned feedback "user_score" score of 1: + + .. code-block:: python + good_extractor_runs = client.list_runs( project_name="", filter='eq(name, "extractor")', trace_filter='and(eq(feedback_key, "user_score"), eq(feedback_score, 1))', ) - # List all runs that started after a specific timestamp and either have "error" not equal to null or a "Correctness" feedback score equal to 0 + List all runs that started after a specific timestamp and either have "error" not equal to null or a "Correctness" feedback score equal to 0: + + .. code-block:: python + complex_runs = client.list_runs( project_name="", filter='and(gt(start_time, "2023-07-15T12:34:56Z"), or(neq(error, null), and(eq(feedback_key, "Correctness"), eq(feedback_score, 0.0))))', ) - # List all runs where `tags` include "experimental" or "beta" and `latency` is greater than 2 seconds + List all runs where `tags` include "experimental" or "beta" and `latency` is greater than 2 seconds: + + .. code-block:: python + tagged_runs = client.list_runs( project_name="", filter='and(or(has(tags, "experimental"), has(tags, "beta")), gt(latency, 2))', ) - """ # noqa: E501 + """ project_ids = [] if isinstance(project_id, (uuid.UUID, str)): project_ids.append(project_id) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index b2dc4ac2b..6377bd2d0 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -1,4 +1,14 @@ -"""The LangSmith Client.""" +"""Client for interacting with the LangSmith API. + +Use the client to customize API keys / workspace ocnnections, SSl certs, +etc. for tracing. + +Also used to create, read, update, and delete LangSmith resources +such as runs (~trace spans), datasets, examples (~records), +feedback (~metrics), projects (tracer sessions/groups), etc. + +For detailed API documentation, visit: https://docs.smith.langchain.com/. +""" from __future__ import annotations @@ -2368,14 +2378,14 @@ def get_test_results( *, project_id: Optional[ID_TYPE] = None, project_name: Optional[str] = None, - ) -> "pd.DataFrame": + ) -> pd.DataFrame: """Read the record-level information from an experiment into a Pandas DF. Note: this will fetch whatever data exists in the DB. Results are not immediately available in the DB upon evaluation run completion. Returns: - ------- + -------- pd.DataFrame A dataframe containing the test results. """ @@ -2715,7 +2725,7 @@ def diff_dataset_versions( Examples: -------- - ..code-block:: python + .. code-block:: python # Get the difference between two tagged versions of a dataset from_version = "prod" @@ -2728,7 +2738,6 @@ def diff_dataset_versions( print(diff) # Get the difference between two timestamped versions of a dataset - from_version = datetime.datetime(2024, 1, 1) to_version = datetime.datetime(2024, 2, 1) diff = client.diff_dataset_versions( @@ -2807,7 +2816,7 @@ def list_datasets( """List the datasets on the LangSmith API. Yields: - ------ + ------- Dataset The datasets. """ @@ -2978,7 +2987,7 @@ def read_dataset_version( Examples: - -------- + --------- .. code-block:: python # Get the latest version of a dataset @@ -3023,11 +3032,6 @@ def clone_public_dataset( Defaults to the API URL of your current client. dataset_name (str): The name of the dataset to create in your tenant. Defaults to the name of the public dataset. - - Returns: - ------- - Dataset - The created dataset. """ source_api_url = source_api_url or self.api_url source_api_url, token_uuid = _parse_token_or_url(token_or_url, source_api_url) @@ -3242,7 +3246,7 @@ def create_examples( The output values for the examples. metadata : Optional[Sequence[Optional[Mapping[str, Any]]]], default=None The metadata for the examples. - split : Optional[Sequence[Optional[str | List[str]]]], default=None + splits : Optional[Sequence[Optional[str | List[str]]]], default=None The splits for the examples, which are divisions of your dataset such as 'train', 'test', or 'validation'. source_run_ids : Optional[Sequence[Optional[ID_TYPE]]], default=None @@ -3253,15 +3257,6 @@ def create_examples( The ID of the dataset to create the examples in. dataset_name : Optional[str], default=None The name of the dataset to create the examples in. - - Returns: - ------- - None - - Raises: - ------ - ValueError - If both `dataset_id` and `dataset_name` are `None`. """ if dataset_id is None and dataset_name is None: raise ValueError("Either dataset_id or dataset_name must be provided.") @@ -3514,7 +3509,7 @@ def similar_examples( r"""Retrieve the dataset examples whose inputs best match the current inputs. **Note**: Must have few-shot indexing enabled for the dataset. See - ``client.index_dataset()``. + `client.index_dataset()`. Args: inputs (dict): The inputs to use as a search query. Must match the dataset @@ -3522,17 +3517,14 @@ def similar_examples( limit (int): The maximum number of examples to return. dataset_id (str or UUID): The ID of the dataset to search over. filter (str, optional): A filter string to apply to the search results. Uses - the same syntax as the `filter` parameter in `list_runs()`. Only a subset - of operations are supported. Defaults to None. - - For example, you can use `and(eq(metadata.some_tag, 'some_value'), neq(metadata.env, 'dev'))` - to filter only examples where some_tag has some_value, and the environment is not dev. - kwargs (Any): Additional keyword args to pass as part of request body. + the same syntax as the `filter` parameter in `list_runs()`. Only a subset + of operations are supported. Defaults to None. - Returns: - List of ExampleSearch objects. + For example, you can use ``and(eq(metadata.some_tag, 'some_value'), neq(metadata.env, 'dev'))`` + to filter only examples where some_tag has some_value, and the environment is not dev. + kwargs (Any): Additional keyword args to pass as part of request body. - Example: + Examples: .. code-block:: python from langsmith import Client @@ -3549,7 +3541,7 @@ def similar_examples( [ ExampleSearch( inputs={'question': 'How do I cache a Chat model? What caches can I use?'}, - outputs={'answer': 'You can use LangChain\'s caching layer for Chat Models. This can save you money by reducing the number of API calls you make to the LLM provider, if you\'re often requesting the same completion multiple times, and speed up your application.\n\n```python\n\nfrom langchain.cache import InMemoryCache\nlangchain.llm_cache = InMemoryCache()\n\n# The first time, it is not yet in cache, so it should take longer\nllm.predict(\'Tell me a joke\')\n\n```\n\nYou can also use SQLite Cache which uses a SQLite database:\n\n```python\n rm .langchain.db\n\nfrom langchain.cache import SQLiteCache\nlangchain.llm_cache = SQLiteCache(database_path=".langchain.db")\n\n# The first time, it is not yet in cache, so it should take longer\nllm.predict(\'Tell me a joke\') \n```\n'}, + outputs={'answer': 'You can use LangChain\'s caching layer for Chat Models. This can save you money by reducing the number of API calls you make to the LLM provider, if you\'re often requesting the same completion multiple times, and speed up your application.\n\nfrom langchain.cache import InMemoryCache\nlangchain.llm_cache = InMemoryCache()\n\n# The first time, it is not yet in cache, so it should take longer\nllm.predict(\'Tell me a joke\')\n\nYou can also use SQLite Cache which uses a SQLite database:\n\nrm .langchain.db\n\nfrom langchain.cache import SQLiteCache\nlangchain.llm_cache = SQLiteCache(database_path=".langchain.db")\n\n# The first time, it is not yet in cache, so it should take longer\nllm.predict(\'Tell me a joke\') \n'}, metadata=None, id=UUID('b2ddd1c4-dff6-49ae-8544-f48e39053398'), dataset_id=UUID('01b6ce0f-bfb6-4f48-bbb8-f19272135d40') @@ -3563,14 +3555,14 @@ def similar_examples( ), ExampleSearch( inputs={'question': 'Show me how to use RecursiveURLLoader'}, - outputs={'answer': 'The RecursiveURLLoader comes from the langchain.document_loaders.recursive_url_loader module. Here\'s an example of how to use it:\n\n```python\nfrom langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader\n\n# Create an instance of RecursiveUrlLoader with the URL you want to load\nloader = RecursiveUrlLoader(url="https://example.com")\n\n# Load all child links from the URL page\nchild_links = loader.load()\n\n# Print the child links\nfor link in child_links:\n print(link)\n```\n\nMake sure to replace "https://example.com" with the actual URL you want to load. The load() method returns a list of child links found on the URL page. You can iterate over this list to access each child link.'}, + outputs={'answer': 'The RecursiveURLLoader comes from the langchain.document_loaders.recursive_url_loader module. Here\'s an example of how to use it:\n\nfrom langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader\n\n# Create an instance of RecursiveUrlLoader with the URL you want to load\nloader = RecursiveUrlLoader(url="https://example.com")\n\n# Load all child links from the URL page\nchild_links = loader.load()\n\n# Print the child links\nfor link in child_links:\n print(link)\n\nMake sure to replace "https://example.com" with the actual URL you want to load. The load() method returns a list of child links found on the URL page. You can iterate over this list to access each child link.'}, metadata=None, id=UUID('0308ea70-a803-4181-a37d-39e95f138f8c'), dataset_id=UUID('01b6ce0f-bfb6-4f48-bbb8-f19272135d40') ), ] - """ # noqa: E501 + """ dataset_id = _as_uuid(dataset_id, "dataset_id") req = { "inputs": inputs, @@ -4093,7 +4085,7 @@ def create_feedback( feedback_id : str or UUID or None, default=None The ID of the feedback to create. If not provided, a random UUID will be generated. - feedback_config: FeedbackConfig or None, default=None, + feedback_config: langsmith.schemas.FeedbackConfig or None, default=None, The configuration specifying how to interpret feedback with this key. Examples include continuous (with min/max bounds), categorical, or freeform. @@ -4786,115 +4778,10 @@ async def arun_on_dataset( ) -> Dict[str, Any]: """Asynchronously run the Chain or language model on a dataset. - Store traces to the specified project name. - - Args: - dataset_name: Name of the dataset to run the chain on. - llm_or_chain_factory: Language model or Chain constructor to run - over the dataset. The Chain constructor is used to permit - independent calls on each example without carrying over state. - evaluation: Optional evaluation configuration to use when evaluating - concurrency_level: The number of async tasks to run concurrently. - project_name: Name of the project to store the traces in. - Defaults to a randomly generated name. - project_metadata: Optional metadata to store with the project. - dataset_version: Optional version identifier to run the dataset on. - Can be a timestamp or a string tag. - verbose: Whether to print progress. - tags: Tags to add to each run in the project. - input_mapper: A function to map to the inputs dictionary from an Example - to the format expected by the model to be evaluated. This is useful if - your model needs to deserialize more complex schema or if your dataset - has inputs with keys that differ from what is expected by your chain - or agent. - revision_id: Optional revision identifier to assign this test run to - track the performance of different versions of your system. - - Returns: - A dictionary containing the run's project name and the - resulting model outputs. - - For the synchronous version, see client.run_on_dataset. - - Examples: - -------- - .. code-block:: python - - from langsmith import Client - from langchain.chat_models import ChatOpenAI - from langchain.chains import LLMChain - from langchain.smith import RunEvalConfig - - - # Chains may have memory. Passing in a constructor function lets the - # evaluation framework avoid cross-contamination between runs. - def construct_chain(): - llm = ChatOpenAI(temperature=0) - chain = LLMChain.from_string(llm, "What's the answer to {your_input_key}") - return chain - - - # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) - evaluation_config = RunEvalConfig( - evaluators=[ - "qa", # "Correctness" against a reference answer - "embedding_distance", - RunEvalConfig.Criteria("helpfulness"), - RunEvalConfig.Criteria( - { - "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" - } - ), - ] - ) - - client = Client() - await client.arun_on_dataset( - "", - construct_chain, - evaluation=evaluation_config, - ) - - You can also create custom evaluators by subclassing the - :class:`StringEvaluator ` - or LangSmith's `RunEvaluator` classes. - - .. code-block:: python - - from typing import Optional - from langchain.evaluation import StringEvaluator - - - class MyStringEvaluator(StringEvaluator): - @property - def requires_input(self) -> bool: - return False - - @property - def requires_reference(self) -> bool: - return True + .. deprecated:: 0.1.0 + This method is deprecated. Use :func:`langsmith.aevaluate` instead. - @property - def evaluation_name(self) -> str: - return "exact_match" - - def _evaluate_strings( - self, prediction, reference=None, input=None, **kwargs - ) -> dict: - return {"score": prediction == reference} - - - evaluation_config = RunEvalConfig( - custom_evaluators=[MyStringEvaluator()], - ) - - await client.arun_on_dataset( - "", - construct_chain, - evaluation=evaluation_config, - ) """ # noqa: E501 - # warn as deprecated and to use `aevaluate` instead warnings.warn( "The `arun_on_dataset` method is deprecated and" " will be removed in a future version." @@ -4940,115 +4827,10 @@ def run_on_dataset( ) -> Dict[str, Any]: """Run the Chain or language model on a dataset. - Store traces to the specified project name. - - Args: - dataset_name: Name of the dataset to run the chain on. - llm_or_chain_factory: Language model or Chain constructor to run - over the dataset. The Chain constructor is used to permit - independent calls on each example without carrying over state. - evaluation: Configuration for evaluators to run on the - results of the chain - concurrency_level: The number of tasks to execute concurrently. - project_name: Name of the project to store the traces in. - Defaults to a randomly generated name. - project_metadata: Metadata to store with the project. - dataset_version: Optional version identifier to run the dataset on. - Can be a timestamp or a string tag. - verbose: Whether to print progress. - tags: Tags to add to each run in the project. - input_mapper: A function to map to the inputs dictionary from an Example - to the format expected by the model to be evaluated. This is useful if - your model needs to deserialize more complex schema or if your dataset - has inputs with keys that differ from what is expected by your chain - or agent. - revision_id: Optional revision identifier to assign this test run to - track the performance of different versions of your system. - - Returns: - A dictionary containing the run's project name and the resulting model outputs. - - - For the (usually faster) async version of this function, see `client.arun_on_dataset`. - - Examples: - -------- - .. code-block:: python - - from langsmith import Client - from langchain.chat_models import ChatOpenAI - from langchain.chains import LLMChain - from langchain.smith import RunEvalConfig - + .. deprecated:: 0.1.0 + This method is deprecated. Use :func:`langsmith.aevaluate` instead. - # Chains may have memory. Passing in a constructor function lets the - # evaluation framework avoid cross-contamination between runs. - def construct_chain(): - llm = ChatOpenAI(temperature=0) - chain = LLMChain.from_string(llm, "What's the answer to {your_input_key}") - return chain - - - # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) - evaluation_config = RunEvalConfig( - evaluators=[ - "qa", # "Correctness" against a reference answer - "embedding_distance", - RunEvalConfig.Criteria("helpfulness"), - RunEvalConfig.Criteria( - { - "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" - } - ), - ] - ) - - client = Client() - client.run_on_dataset( - "", - construct_chain, - evaluation=evaluation_config, - ) - - You can also create custom evaluators by subclassing the - :class:`StringEvaluator ` - or LangSmith's `RunEvaluator` classes. - - .. code-block:: python - - from typing import Optional - from langchain.evaluation import StringEvaluator - - - class MyStringEvaluator(StringEvaluator): - @property - def requires_input(self) -> bool: - return False - - @property - def requires_reference(self) -> bool: - return True - - @property - def evaluation_name(self) -> str: - return "exact_match" - - def _evaluate_strings( - self, prediction, reference=None, input=None, **kwargs - ) -> dict: - return {"score": prediction == reference} - - - evaluation_config = RunEvalConfig( - custom_evaluators=[MyStringEvaluator()], - ) - - client.run_on_dataset( - "", - construct_chain, - evaluation=evaluation_config, - ) - """ # noqa: E501 + """ # noqa: E501 # noqa: E501 warnings.warn( "The `run_on_dataset` method is deprecated and" " will be removed in a future version." diff --git a/python/langsmith/evaluation/__init__.py b/python/langsmith/evaluation/__init__.py index 253732cfc..244f9a7d8 100644 --- a/python/langsmith/evaluation/__init__.py +++ b/python/langsmith/evaluation/__init__.py @@ -24,7 +24,6 @@ def __getattr__(name: str) -> Any: - # TODO: Use importlib if name == "evaluate": from langsmith.evaluation._runner import evaluate diff --git a/python/langsmith/evaluation/_runner.py b/python/langsmith/evaluation/_runner.py index 45478ad2d..b57d18753 100644 --- a/python/langsmith/evaluation/_runner.py +++ b/python/langsmith/evaluation/_runner.py @@ -1061,7 +1061,9 @@ def _print_experiment_start( ) else: # HACKHACK - print("Starting evaluation of experiment: %s", self.experiment_name) + print( # noqa: T201 + "Starting evaluation of experiment: %s", self.experiment_name + ) class _ExperimentManager(_ExperimentManagerMixin): diff --git a/python/langsmith/run_helpers.py b/python/langsmith/run_helpers.py index dbbae0904..c08f4874c 100644 --- a/python/langsmith/run_helpers.py +++ b/python/langsmith/run_helpers.py @@ -204,16 +204,27 @@ class LangSmithExtra(TypedDict, total=False): """Any additional info to be injected into the run dynamically.""" name: Optional[str] + """Optional name for the run.""" reference_example_id: Optional[ls_client.ID_TYPE] + """Optional ID of a reference example.""" run_extra: Optional[Dict] + """Optional additional run information.""" parent: Optional[Union[run_trees.RunTree, str, Mapping]] + """Optional parent run, can be a RunTree, string, or mapping.""" run_tree: Optional[run_trees.RunTree] # TODO: Deprecate + """Optional run tree (deprecated).""" project_name: Optional[str] + """Optional name of the project.""" metadata: Optional[Dict[str, Any]] + """Optional metadata for the run.""" tags: Optional[List[str]] + """Optional list of tags for the run.""" run_id: Optional[ls_client.ID_TYPE] + """Optional ID for the run.""" client: Optional[ls_client.Client] + """Optional LangSmith client.""" on_end: Optional[Callable[[run_trees.RunTree], Any]] + """Optional callback function to be called when the run ends.""" R = TypeVar("R", covariant=True) @@ -293,9 +304,9 @@ def traceable( None, which will use the default client. reduce_fn: A function to reduce the output of the function if the function returns a generator. Defaults to None, which means the values will be - logged as a list. Note: if the iterator is never exhausted (e.g. - the function returns an infinite generator), this will never be - called, and the run itself will be stuck in a pending state. + logged as a list. Note: if the iterator is never exhausted (e.g. + the function returns an infinite generator), this will never be + called, and the run itself will be stuck in a pending state. project_name: The name of the project to log the run to. Defaults to None, which will use the default project. process_inputs: Custom serialization / processing function for inputs. @@ -303,8 +314,6 @@ def traceable( process_outputs: Custom serialization / processing function for outputs. Defaults to None. - - Returns: Union[Callable, Callable[[Callable], Callable]]: The decorated function. @@ -312,15 +321,10 @@ def traceable( - Requires that LANGSMITH_TRACING_V2 be set to 'true' in the environment. Examples: - .. code-block:: python - import httpx - import asyncio - - from typing import Iterable - from langsmith import traceable, Client + Basic usage: + .. code-block:: python - # Basic usage: @traceable def my_function(x: float, y: float) -> float: return x + y @@ -341,8 +345,10 @@ async def my_async_function(query_params: dict) -> dict: asyncio.run(my_async_function({"param": "value"})) + Streaming data with a generator: + + .. code-block:: python - # Streaming data with a generator: @traceable def my_generator(n: int) -> Iterable: for i in range(n): @@ -352,8 +358,10 @@ def my_generator(n: int) -> Iterable: for item in my_generator(5): print(item) + Async streaming data: + + .. code-block:: python - # Async streaming data @traceable async def my_async_generator(query_params: dict) -> Iterable: async with httpx.AsyncClient() as http_client: @@ -372,8 +380,10 @@ async def async_code(): asyncio.run(async_code()) + Specifying a run type and name: + + .. code-block:: python - # Specifying a run type and name: @traceable(name="CustomName", run_type="tool") def another_function(a: float, b: float) -> float: return a * b @@ -381,8 +391,10 @@ def another_function(a: float, b: float) -> float: another_function(5, 6) + Logging with custom metadata and tags: + + .. code-block:: python - # Logging with custom metadata and tags: @traceable( metadata={"version": "1.0", "author": "John Doe"}, tags=["beta", "test"] ) @@ -392,7 +404,10 @@ def tagged_function(x): tagged_function(5) - # Specifying a custom client and project name: + Specifying a custom client and project name: + + .. code-block:: python + custom_client = Client(api_key="your_api_key") @@ -403,15 +418,17 @@ def project_specific_function(data): project_specific_function({"data": "to process"}) + Manually passing langsmith_extra: + + .. code-block:: python - # Manually passing langsmith_extra: @traceable def manual_extra_function(x): return x**2 manual_extra_function(5, langsmith_extra={"metadata": {"version": "1.0"}}) - """ # noqa: E501 + """ run_type: ls_client.RUN_TYPE_T = ( args[0] if args and isinstance(args[0], str) @@ -742,64 +759,57 @@ def generator_wrapper( class trace: - """Manage a langsmith run in context. + """Manage a LangSmith run in context. This class can be used as both a synchronous and asynchronous context manager. - Parameters: - ----------- - name : str - Name of the run - run_type : ls_client.RUN_TYPE_T, optional - Type of run (e.g., "chain", "llm", "tool"). Defaults to "chain". - inputs : Optional[Dict], optional - Initial input data for the run - project_name : Optional[str], optional - Associates the run with a specific project, overriding defaults - parent : Optional[Union[run_trees.RunTree, str, Mapping]], optional - Parent run, accepts RunTree, dotted order string, or tracing headers - tags : Optional[List[str]], optional - Categorization labels for the run - metadata : Optional[Mapping[str, Any]], optional - Arbitrary key-value pairs for run annotation - client : Optional[ls_client.Client], optional - LangSmith client for specifying a different tenant, - setting custom headers, or modifying API endpoint - run_id : Optional[ls_client.ID_TYPE], optional - Preset identifier for the run - reference_example_id : Optional[ls_client.ID_TYPE], optional - You typically won't set this. It associates this run with a dataset example. - This is only valid for root runs (not children) in an evaluation context. - exceptions_to_handle : Optional[Tuple[Type[BaseException], ...]], optional - Typically not set. Exception types to ignore in what is sent up to LangSmith - extra : Optional[Dict], optional - Typically not set. Use 'metadata' instead. Extra data to be sent to LangSmith. + Args: + name (str): Name of the run. + run_type (ls_client.RUN_TYPE_T, optional): Type of run (e.g., "chain", "llm", "tool"). Defaults to "chain". + inputs (Optional[Dict], optional): Initial input data for the run. Defaults to None. + project_name (Optional[str], optional): Project name to associate the run with. Defaults to None. + parent (Optional[Union[run_trees.RunTree, str, Mapping]], optional): Parent run. Can be a RunTree, dotted order string, or tracing headers. Defaults to None. + tags (Optional[List[str]], optional): List of tags for the run. Defaults to None. + metadata (Optional[Mapping[str, Any]], optional): Additional metadata for the run. Defaults to None. + client (Optional[ls_client.Client], optional): LangSmith client for custom settings. Defaults to None. + run_id (Optional[ls_client.ID_TYPE], optional): Preset identifier for the run. Defaults to None. + reference_example_id (Optional[ls_client.ID_TYPE], optional): Associates run with a dataset example. Only for root runs in evaluation. Defaults to None. + exceptions_to_handle (Optional[Tuple[Type[BaseException], ...]], optional): Exception types to ignore. Defaults to None. + extra (Optional[Dict], optional): Extra data to send to LangSmith. Use 'metadata' instead. Defaults to None. Examples: - --------- - Synchronous usage: - >>> with trace("My Operation", run_type="tool", tags=["important"]) as run: - ... result = "foo" # Do some_operation() - ... run.metadata["some-key"] = "some-value" - ... run.end(outputs={"result": result}) - - Asynchronous usage: - >>> async def main(): - ... async with trace("Async Operation", run_type="tool", tags=["async"]) as run: - ... result = "foo" # Can await some_async_operation() - ... run.metadata["some-key"] = "some-value" - ... # "end" just adds the outputs and sets error to None - ... # The actual patching of the run happens when the context exits - ... run.end(outputs={"result": result}) - >>> asyncio.run(main()) - - Allowing pytest.skip in a test: - >>> import sys - >>> import pytest - >>> with trace("OS-Specific Test", exceptions_to_handle=(pytest.skip.Exception,)): - ... if sys.platform == "win32": - ... pytest.skip("Not supported on Windows") - ... result = "foo" # e.g., do some unix_specific_operation() + Synchronous usage: + + .. code-block:: python + + >>> with trace("My Operation", run_type="tool", tags=["important"]) as run: + ... result = "foo" # Perform operation + ... run.metadata["some-key"] = "some-value" + ... run.end(outputs={"result": result}) + + Asynchronous usage: + + .. code-block:: python + + >>> async def main(): + ... async with trace("Async Operation", run_type="tool", tags=["async"]) as run: + ... result = "foo" # Await async operation + ... run.metadata["some-key"] = "some-value" + ... # "end" just adds the outputs and sets error to None + ... # The actual patching of the run happens when the context exits + ... run.end(outputs={"result": result}) + >>> asyncio.run(main()) + + Handling specific exceptions: + + .. code-block:: python + + >>> import pytest + >>> import sys + >>> with trace("Test", exceptions_to_handle=(pytest.skip.Exception,)): + ... if sys.platform == "win32": # Just an example + ... pytest.skip("Skipping test for windows") + ... result = "foo" # Perform test operation """ def __init__( diff --git a/python/langsmith/run_trees.py b/python/langsmith/run_trees.py index 238497036..329cd3a7a 100644 --- a/python/langsmith/run_trees.py +++ b/python/langsmith/run_trees.py @@ -169,8 +169,7 @@ def add_event( events (Union[ls_schemas.RunEvent, Sequence[ls_schemas.RunEvent], Sequence[dict], dict, str]): The event(s) to be added. It can be a single event, a sequence - of events, - a sequence of dictionaries, a dictionary, or a string. + of events, a sequence of dictionaries, a dictionary, or a string. Returns: None diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 34711e20a..4985109d1 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -308,8 +308,8 @@ class Run(RunBase): sorted in the order it was executed. Example: - - Parent: 20230914T223155647Z1b64098b-4ab7-43f6-afee-992304f198d8 - - Children: + - Parent: 20230914T223155647Z1b64098b-4ab7-43f6-afee-992304f198d8 + - Children: - 20230914T223155647Z1b64098b-4ab7-43f6-afee-992304f198d8.20230914T223155649Z809ed3a2-0172-4f4d-8a02-a64e9b7a0f8a - 20230915T223155647Z1b64098b-4ab7-43f6-afee-992304f198d8.20230914T223155650Zc8d9f4c5-6c5a-4b2d-9b1c-3d9d7a7c5c7c """ # noqa: E501 @@ -389,15 +389,12 @@ class FeedbackSourceBase(BaseModel): This represents whether feedback is submitted from the API, model, human labeler, etc. - - Attributes: - type (str): The type of the feedback source. - metadata (Optional[Dict[str, Any]]): Additional metadata for the feedback - source. """ type: str + """The type of the feedback source.""" metadata: Optional[Dict[str, Any]] = Field(default_factory=dict) + """Additional metadata for the feedback source.""" class APIFeedbackSource(FeedbackSourceBase): @@ -463,25 +460,23 @@ class FeedbackCategory(TypedDict, total=False): """Specific value and label pair for feedback.""" value: float + """The numeric value associated with this feedback category.""" label: Optional[str] + """An optional label to interpret the value for this feedback category.""" class FeedbackConfig(TypedDict, total=False): - """Represents _how_ a feedback value ought to be interpreted. - - Attributes: - type (Literal["continuous", "categorical", "freeform"]): The type of feedback. - min (Optional[float]): The minimum value for continuous feedback. - max (Optional[float]): The maximum value for continuous feedback. - categories (Optional[List[FeedbackCategory]]): If feedback is categorical, - This defines the valid categories the server will accept. - Not applicable to continuosu or freeform feedback types. - """ + """Represents _how_ a feedback value ought to be interpreted.""" type: Literal["continuous", "categorical", "freeform"] + """The type of feedback.""" min: Optional[float] + """The minimum value for continuous feedback.""" max: Optional[float] + """The maximum value for continuous feedback.""" categories: Optional[List[FeedbackCategory]] + """If feedback is categorical, this defines the valid categories the server will accept. + Not applicable to continuous or freeform feedback types.""" # noqa class FeedbackCreate(FeedbackBase): @@ -599,7 +594,9 @@ class BaseMessageLike(Protocol): """A protocol representing objects similar to BaseMessage.""" content: str - additional_kwargs: Dict + """The content of the message.""" + additional_kwargs: Dict[Any, Any] + """Additional keyword arguments associated with the message.""" @property def type(self) -> str: @@ -607,58 +604,46 @@ def type(self) -> str: class DatasetShareSchema(TypedDict, total=False): - """Represents the schema for a dataset share. - - Attributes: - dataset_id (UUID): The ID of the dataset. - share_token (UUID): The token for sharing the dataset. - url (str): The URL of the shared dataset. - """ + """Represents the schema for a dataset share.""" dataset_id: UUID + """The ID of the dataset.""" share_token: UUID + """The token for sharing the dataset.""" url: str + """The URL of the shared dataset.""" class AnnotationQueue(BaseModel): - """Represents an annotation queue. - - Attributes: - id (UUID): The ID of the annotation queue. - name (str): The name of the annotation queue. - description (Optional[str], optional): The description of the annotation queue. - Defaults to None. - created_at (datetime, optional): The creation timestamp of the annotation queue. - Defaults to the current UTC time. - updated_at (datetime, optional): The last update timestamp of the annotation - queue. Defaults to the current UTC time. - tenant_id (UUID): The ID of the tenant associated with the annotation queue. - """ + """Represents an annotation queue.""" id: UUID + """The unique identifier of the annotation queue.""" name: str + """The name of the annotation queue.""" description: Optional[str] = None + """An optional description of the annotation queue.""" created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + """The timestamp when the annotation queue was created.""" updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + """The timestamp when the annotation queue was last updated.""" tenant_id: UUID + """The ID of the tenant associated with the annotation queue.""" class BatchIngestConfig(TypedDict, total=False): - """Configuration for batch ingestion. - - Attributes: - scale_up_qsize_trigger (int): The queue size threshold that triggers scaling up. - scale_up_nthreads_limit (int): The maximum number of threads to scale up to. - scale_down_nempty_trigger (int): The number of empty threads that triggers - scaling down. - size_limit (int): The maximum size limit for the batch. - """ + """Configuration for batch ingestion.""" scale_up_qsize_trigger: int + """The queue size threshold that triggers scaling up.""" scale_up_nthreads_limit: int + """The maximum number of threads to scale up to.""" scale_down_nempty_trigger: int + """The number of empty threads that triggers scaling down.""" size_limit: int + """The maximum size limit for the batch.""" size_limit_bytes: Optional[int] + """The maximum size limit in bytes for the batch.""" class LangSmithInfo(BaseModel): @@ -687,17 +672,14 @@ class LangSmithSettings(BaseModel): class FeedbackIngestToken(BaseModel): - """Represents the schema for a feedback ingest token. - - Attributes: - id (UUID): The ID of the feedback ingest token. - token (str): The token for ingesting feedback. - expires_at (datetime): The expiration time of the token. - """ + """Represents the schema for a feedback ingest token.""" id: UUID + """The ID of the feedback ingest token.""" url: str + """The URL to GET when logging the feedback.""" expires_at: datetime + """The expiration time of the token.""" class RunEvent(TypedDict, total=False): @@ -723,20 +705,14 @@ class TimeDeltaInput(TypedDict, total=False): class DatasetDiffInfo(BaseModel): - """Represents the difference information between two datasets. - - Attributes: - examples_modified (List[UUID]): A list of UUIDs representing - the modified examples. - examples_added (List[UUID]): A list of UUIDs representing - the added examples. - examples_removed (List[UUID]): A list of UUIDs representing - the removed examples. - """ + """Represents the difference information between two datasets.""" examples_modified: List[UUID] + """A list of UUIDs representing the modified examples.""" examples_added: List[UUID] + """A list of UUIDs representing the added examples.""" examples_removed: List[UUID] + """A list of UUIDs representing the removed examples.""" class ComparativeExperiment(BaseModel): @@ -747,15 +723,25 @@ class ComparativeExperiment(BaseModel): """ id: UUID + """The unique identifier for the comparative experiment.""" name: Optional[str] = None + """The optional name of the comparative experiment.""" description: Optional[str] = None + """An optional description of the comparative experiment.""" tenant_id: UUID + """The identifier of the tenant associated with this experiment.""" created_at: datetime + """The timestamp when the comparative experiment was created.""" modified_at: datetime + """The timestamp when the comparative experiment was last modified.""" reference_dataset_id: UUID + """The identifier of the reference dataset used in this experiment.""" extra: Optional[Dict[str, Any]] = None + """Optional additional information about the experiment.""" experiments_info: Optional[List[dict]] = None + """Optional list of dictionaries containing information about individual experiments.""" feedback_stats: Optional[Dict[str, Any]] = None + """Optional dictionary containing feedback statistics for the experiment.""" @property def metadata(self) -> dict[str, Any]: @@ -766,15 +752,7 @@ def metadata(self) -> dict[str, Any]: class PromptCommit(BaseModel): - """Represents a Prompt with a manifest. - - Attributes: - owner (str): The handle of the owner of the prompt. - repo (str): The name of the prompt. - commit_hash (str): The commit hash of the prompt. - manifest (Dict[str, Any]): The manifest of the prompt. - examples (List[dict]): The list of examples. - """ + """Represents a Prompt with a manifest.""" owner: str """The handle of the owner of the prompt.""" diff --git a/python/langsmith/utils.py b/python/langsmith/utils.py index bf9d17b68..5530dcf2f 100644 --- a/python/langsmith/utils.py +++ b/python/langsmith/utils.py @@ -676,7 +676,7 @@ def map( ) -> Iterator[T]: """Return an iterator equivalent to stdlib map. - Each function will receive it's own copy of the context from the parent thread. + Each function will receive its own copy of the context from the parent thread. Args: fn: A callable that will take as many arguments as there are diff --git a/python/pyproject.toml b/python/pyproject.toml index ac4d8cb86..8902a3d37 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -79,8 +79,12 @@ lint.select = [ "I", # isort "D", # pydocstyle "D401", # First line should be in imperative mood + "T201", + "UP", ] lint.ignore = [ + "UP006", + "UP007", # Relax the convention by _not_ requiring documentation for every function parameter. "D417", ] @@ -88,8 +92,18 @@ lint.ignore = [ convention = "google" [tool.ruff.lint.per-file-ignores] -"tests/*" = ["D"] -"langsmith/cli/*" = ["D"] +"langsmith/run_helpers.py" = ["E501"] +"docs/conf.py" = ["E501"] +"langsmith/cli/*" = ["T201", "D", "UP"] +"docs/create_api_rst.py" = ["D101", "D103"] +"docs/scripts/custom_formatter.py" = ["D100"] +"langsmith/anonymizer.py" = ["E501"] +"langsmith/async_client.py" = ["E501"] +"langsmith/client.py" = ["E501"] +"langsmith/schemas.py" = ["E501"] +"tests/evaluation/__init__.py" = ["E501"] +"tests/*" = ["D", "UP"] +"docs/*" = ["T", "D"] [tool.ruff.format] docstring-code-format = true @@ -102,4 +116,4 @@ disallow_untyped_defs = "True" [tool.pytest.ini_options] asyncio_mode = "auto" -markers = [ "slow: long-running tests",] +markers = ["slow: long-running tests"] diff --git a/python/tests/evaluation/__init__.py b/python/tests/evaluation/__init__.py index e69de29bb..f2f869cab 100644 --- a/python/tests/evaluation/__init__.py +++ b/python/tests/evaluation/__init__.py @@ -0,0 +1,31 @@ +"""LangSmith Evaluations. + +This module provides a comprehensive suite of tools for evaluating language models and their outputs using LangSmith. + +Key Features: +- Robust evaluation framework for assessing model performance across diverse tasks +- Flexible configuration options for customizing evaluation criteria and metrics +- Seamless integration with LangSmith's platform for end-to-end evaluation workflows +- Advanced analytics and reporting capabilities for actionable insights + +Usage: +1. Import the necessary components from this module +2. Configure your evaluation parameters and criteria +3. Run your language model through the evaluation pipeline +4. Analyze the results using our built-in tools or export for further processing + +Example: + from langsmith.evaluation import RunEvaluator, MetricCalculator + + evaluator = RunEvaluator(model="gpt-3.5-turbo", dataset_name="customer_support") + results = evaluator.run() + metrics = MetricCalculator(results).calculate() + + print(metrics.summary()) + +For detailed API documentation and advanced usage scenarios, visit: +https://docs.langsmith.com/evaluation + +Note: This module is designed to work seamlessly with the LangSmith platform. +Ensure you have the necessary credentials and permissions set up before use. +"""