fix: standardize some things (#157)

langchain-ai · Apr 5, 2024 · 8c6a8da · 8c6a8da
1 parent bdfe252
commit 8c6a8da
Show file tree

Hide file tree

Showing 12 changed files with 327 additions and 253 deletions.
diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml
@@ -1,78 +1,79 @@
-name: release-please
+name: release
+run-name: Release ${{ inputs.working-directory }} by @${{ github.actor }}
 on:
-  push:
-    branches:
-      - main
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+  workflow_dispatch:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        default: '.'
 
 env:
-  PYTHON_VERSION: "3.10"
+  PYTHON_VERSION: "3.11"
   POETRY_VERSION: "1.7.1"
 
 jobs:
-  release-please:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    outputs:
-      releases_created: ${{ steps.release.outputs.release_created }}
-      paths_released: ${{ steps.release.outputs.paths_released }}
-      prs_created: ${{ steps.release.outputs.prs_created }}
-      prs: ${{ steps.release.outputs.prs }}
-    steps:
-      - uses: google-github-actions/release-please-action@v4
-        id: release
-        with:
-          manifest-file: ".release-please-manifest.json"
-          config-file: "release-please-config.json"
   build:
-    needs: release-please
-    if: ${{ needs.release-please.outputs.releases_created }}
+    if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
+
+    outputs:
+      pkg-name: ${{ steps.check-version.outputs.pkg-name }}
+      version: ${{ steps.check-version.outputs.version }}
+
     steps:
       - uses: actions/checkout@v4
+
       - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           poetry-version: ${{ env.POETRY_VERSION }}
-          working-directory: .
+          working-directory: ${{ inputs.working-directory }}
           cache-key: release
+
+      # We want to keep this build stage *separate* from the release stage,
+      # so that there's no sharing of permissions between them.
+      # The release stage has trusted publishing and GitHub repo contents write access,
+      # and we want to keep the scope of that access limited just to the release job.
+      # Otherwise, a malicious `build` step (e.g. via a compromised dependency)
+      # could get access to our GitHub or PyPI credentials.
+      #
+      # Per the trusted publishing GitHub Action:
+      # > It is strongly advised to separate jobs for building [...]
+      # > from the publish job.
+      # https://github.com/pypa/gh-action-pypi-publish#non-goals
       - name: Build project for distribution
         run: poetry build
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
 
       - name: Upload build
         uses: actions/upload-artifact@v3
         with:
           name: dist
-          path: ./dist/
+          path: ${{ inputs.working-directory }}/dist/
 
       - name: Check Version
         id: check-version
         shell: bash
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
         run: |
           echo pkg-name="$(poetry version | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
           echo version="$(poetry version --short)" >> $GITHUB_OUTPUT
 
-      # We want to keep this build stage *separate* from the release stage,
-      # so that there's no sharing of permissions between them.
-      # The release stage has trusted publishing and GitHub repo contents write access,
-      # and we want to keep the scope of that access limited just to the release job.
-      # Otherwise, a malicious `build` step (e.g. via a compromised dependency)
-      # could get access to our GitHub or PyPI credentials.
-      #
-      # Per the trusted publishing GitHub Action:
-      # > It is strongly advised to separate jobs for building [...]
-      # > from the publish job.
-      # https://github.com/pypa/gh-action-pypi-publish#non-goals
-
   test-pypi-publish:
     needs:
       - build
     uses:
       ./.github/workflows/_test_release.yml
+    with:
+      working-directory: ${{ inputs.working-directory }}
     secrets: inherit
 
   pre-release-checks:
@@ -101,11 +102,11 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           poetry-version: ${{ env.POETRY_VERSION }}
-          working-directory: .
+          working-directory: ${{ inputs.working-directory }}
 
       - name: Import published package
         shell: bash
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
         env:
           PKG_NAME: ${{ needs.build.outputs.pkg-name }}
           VERSION: ${{ needs.build.outputs.version }}
@@ -137,11 +138,11 @@ jobs:
 
       - name: Import test dependencies
         run: poetry install --with test,test_integration
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
 
       # Overwrite the local version of the package with the test PyPI version.
       - name: Import published package (again)
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
         shell: bash
         env:
           PKG_NAME: ${{ needs.build.outputs.pkg-name }}
@@ -153,26 +154,33 @@ jobs:
 
       - name: Run unit tests
         run: make tests
-        working-directory: .
-
-      - name: 'Authenticate to Google Cloud'
-        id: 'auth'
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        working-directory: ${{ inputs.working-directory }}
 
       - name: Run integration tests
         env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ES_API_KEY: ${{ secrets.ES_API_KEY }}
+          ES_CLOUD_ID: ${{ secrets.ES_CLOUD_ID }}
+          ES_URL: ${{ secrets.ES_URL }}
         run: make integration_tests
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
+
+      - name: Get minimum versions
+        working-directory: ${{ inputs.working-directory }}
+        id: min-version
+        run: |
+          poetry run pip install packaging
+          min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml)"
+          echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT"
+          echo "min-versions=$min_versions"
 
-      # - name: Run unit tests with minimum dependency versions
-      #   if: ${{ (inputs.working-directory == 'libs/langchain') || (inputs.working-directory == 'libs/community') || (inputs.working-directory == 'libs/experimental') }}
-      #   run: |
-      #     poetry run pip install -r _test_minimum_requirements.txt
-      #     make tests
-      #   working-directory: .
+      - name: Run unit tests with minimum dependency versions
+        if: ${{ steps.min-version.outputs.min-versions != '' }}
+        env:
+          MIN_VERSIONS: ${{ steps.min-version.outputs.min-versions }}
+        run: |
+          poetry run pip install $MIN_VERSIONS
+          make tests
+        working-directory: ${{ inputs.working-directory }}
 
   publish:
     needs:
@@ -190,7 +198,7 @@ jobs:
 
     defaults:
       run:
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
 
     steps:
       - uses: actions/checkout@v4
@@ -200,18 +208,18 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           poetry-version: ${{ env.POETRY_VERSION }}
-          working-directory: .
+          working-directory: ${{ inputs.working-directory }}
           cache-key: release
 
       - uses: actions/download-artifact@v3
         with:
           name: dist
-          path: ./dist/
+          path: ${{ inputs.working-directory }}/dist/
 
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          packages-dir: ./dist/
+          packages-dir: ${{ inputs.working-directory }}/dist/
           verbose: true
           print-hash: true
 
@@ -229,7 +237,7 @@ jobs:
 
     defaults:
       run:
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
 
     steps:
       - uses: actions/checkout@v4
@@ -239,21 +247,20 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           poetry-version: ${{ env.POETRY_VERSION }}
-          working-directory: .
+          working-directory: ${{ inputs.working-directory }}
           cache-key: release
 
       - uses: actions/download-artifact@v3
         with:
           name: dist
-          path: ./dist/
-
-      # - name: Create Release
-      #   uses: ncipollo/release-action@v1
-      #   if: ${{ inputs.working-directory == 'libs/langchain' }}
-      #   with:
-      #     artifacts: "dist/*"
-      #     token: ${{ secrets.GITHUB_TOKEN }}
-      #     draft: false
-      #     generateReleaseNotes: true
-      #     tag: v${{ needs.build.outputs.version }}
-      #     commit: master
+          path: ${{ inputs.working-directory }}/dist/
+
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "dist/*"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: false
+          generateReleaseNotes: true
+          tag: ${{ inputs.working-directory }}/v${{ needs.build.outputs.version }}
+          commit: main
diff --git a/.github/workflows/_test_release.yml b/.github/workflows/_test_release.yml
@@ -2,6 +2,11 @@ name: test-release
 
 on:
   workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
 
 env:
   POETRY_VERSION: "1.7.1"
@@ -24,7 +29,7 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           poetry-version: ${{ env.POETRY_VERSION }}
-          working-directory: .
+          working-directory: ${{ inputs.working-directory }}
           cache-key: release
 
       # We want to keep this build stage *separate* from the release stage,
@@ -40,18 +45,18 @@ jobs:
       # https://github.com/pypa/gh-action-pypi-publish#non-goals
       - name: Build project for distribution
         run: poetry build
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
 
       - name: Upload build
         uses: actions/upload-artifact@v3
         with:
           name: test-dist
-          path: ./dist/
+          path: ${{ inputs.working-directory }}/dist/
 
       - name: Check Version
         id: check-version
         shell: bash
-        working-directory: .
+        working-directory: ${{ inputs.working-directory }}
         run: |
           echo pkg-name="$(poetry version | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
           echo version="$(poetry version --short)" >> $GITHUB_OUTPUT
@@ -74,12 +79,12 @@ jobs:
       - uses: actions/download-artifact@v3
         with:
           name: test-dist
-          path: ./dist/
+          path: ${{ inputs.working-directory }}/dist/
 
       - name: Publish to test PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          packages-dir: ./dist/
+          packages-dir: ${{ inputs.working-directory }}/dist/
           verbose: true
           print-hash: true
           repository-url: https://test.pypi.org/legacy/

diff --git a/Makefile b/Makefile
@@ -6,7 +6,7 @@ all: help
 # Define a variable for the test file path.
 TEST_FILE ?= tests/unit_tests/
 
-integration_tests: TEST_FILE=tests/integration_tests/
+integration_test integration_tests: TEST_FILE=tests/integration_tests/
 
 # test:
 # 	poetry run pytest $(TEST_FILE)
@@ -31,12 +31,8 @@ update-weaviate-image:
 		echo "No update required. Current Weaviate version is already $(LATEST_VERSION)"; \
 	fi
 
-test: update-weaviate-image
-	poetry run pytest -n `nproc` --cov=langchain_weaviate --cov-report term-missing
-
-tests integration_tests:
-	poetry run pytest $(TEST_FILE)
-
+test tests integration_test integration_tests: update-weaviate-image
+	poetry run pytest $(TEST_FILE) --cov=langchain_weaviate --cov-report term-missing
 
 ######################
 # LINTING AND FORMATTING
@@ -46,20 +42,20 @@ tests integration_tests:
 PYTHON_FILES=.
 MYPY_CACHE=.mypy_cache
 lint format: PYTHON_FILES=.
-lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=libs/partners/weaviate --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
 lint_package: PYTHON_FILES=langchain_weaviate
 lint_tests: PYTHON_FILES=tests
 lint_tests: MYPY_CACHE=.mypy_cache_test
 
 lint lint_diff lint_package lint_tests:
-	poetry run ruff .
+	poetry run ruff check .
 	poetry run ruff format $(PYTHON_FILES) --diff
-	poetry run ruff --select I $(PYTHON_FILES)
+	poetry run ruff check --select I $(PYTHON_FILES)
 	mkdir $(MYPY_CACHE); poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE)
 
 format format_diff:
 	poetry run ruff format $(PYTHON_FILES)
-	poetry run ruff --fix $(PYTHON_FILES)
+	poetry run ruff check --fix $(PYTHON_FILES)
 
 spell_check:
 	poetry run codespell --toml pyproject.toml

diff --git a/langchain_weaviate/_math.py b/langchain_weaviate/_math.py
@@ -4,7 +4,7 @@
 from typing import List, Optional, Tuple, Union
 
 import numpy as np
-import simsimd
+import simsimd  # type: ignore
 
 logger = logging.getLogger(__name__)
 

diff --git a/langchain_weaviate/utils.py b/langchain_weaviate/utils.py
@@ -5,7 +5,7 @@
 
 import numpy as np
 
-from ._math import cosine_similarity
+from langchain_weaviate._math import cosine_similarity
 
 
 class DistanceStrategy(str, Enum):