Release libs/core by @baskaryan #756
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: release | |
| run-name: Release ${{ inputs.working-directory }} by @${{ github.actor }} | |
| on: | |
| workflow_call: | |
| inputs: | |
| working-directory: | |
| required: true | |
| type: string | |
| description: "From which folder this pipeline executes" | |
| workflow_dispatch: | |
| inputs: | |
| working-directory: | |
| required: true | |
| type: string | |
| default: 'libs/langchain' | |
| dangerous-nonmaster-release: | |
| required: false | |
| type: boolean | |
| default: false | |
| description: "Release from a non-master branch (danger!)" | |
| env: | |
| PYTHON_VERSION: "3.11" | |
| POETRY_VERSION: "1.7.1" | |
| jobs: | |
| build: | |
| if: github.ref == 'refs/heads/master' || inputs.dangerous-nonmaster-release | |
| environment: Scheduled testing | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pkg-name: ${{ steps.check-version.outputs.pkg-name }} | |
| version: ${{ steps.check-version.outputs.version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} | |
| uses: "./.github/actions/poetry_setup" | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| working-directory: ${{ inputs.working-directory }} | |
| cache-key: release | |
| # We want to keep this build stage *separate* from the release stage, | |
| # so that there's no sharing of permissions between them. | |
| # The release stage has trusted publishing and GitHub repo contents write access, | |
| # and we want to keep the scope of that access limited just to the release job. | |
| # Otherwise, a malicious `build` step (e.g. via a compromised dependency) | |
| # could get access to our GitHub or PyPI credentials. | |
| # | |
| # Per the trusted publishing GitHub Action: | |
| # > It is strongly advised to separate jobs for building [...] | |
| # > from the publish job. | |
| # https://github.com/pypa/gh-action-pypi-publish#non-goals | |
| - name: Build project for distribution | |
| run: poetry build | |
| working-directory: ${{ inputs.working-directory }} | |
| - name: Upload build | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist | |
| path: ${{ inputs.working-directory }}/dist/ | |
| - name: Check Version | |
| id: check-version | |
| shell: bash | |
| working-directory: ${{ inputs.working-directory }} | |
| run: | | |
| echo pkg-name="$(poetry version | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT | |
| echo version="$(poetry version --short)" >> $GITHUB_OUTPUT | |
| release-notes: | |
| needs: | |
| - build | |
| runs-on: ubuntu-latest | |
| outputs: | |
| release-body: ${{ steps.generate-release-body.outputs.release-body }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: langchain-ai/langchain | |
| path: langchain | |
| sparse-checkout: | # this only grabs files for relevant dir | |
| ${{ inputs.working-directory }} | |
| ref: ${{ github.ref }} # this scopes to just ref'd branch | |
| fetch-depth: 0 # this fetches entire commit history | |
| - name: Check Tags | |
| id: check-tags | |
| shell: bash | |
| working-directory: langchain/${{ inputs.working-directory }} | |
| env: | |
| PKG_NAME: ${{ needs.build.outputs.pkg-name }} | |
| VERSION: ${{ needs.build.outputs.version }} | |
| run: | | |
| REGEX="^$PKG_NAME==\\d+\\.\\d+\\.\\d+\$" | |
| echo $REGEX | |
| PREV_TAG=$(git tag --sort=-creatordate | grep -P $REGEX || true | head -1) | |
| TAG="${PKG_NAME}==${VERSION}" | |
| if [ "$TAG" == "$PREV_TAG" ]; then | |
| echo "No new version to release" | |
| exit 1 | |
| fi | |
| echo tag="$TAG" >> $GITHUB_OUTPUT | |
| echo prev-tag="$PREV_TAG" >> $GITHUB_OUTPUT | |
| - name: Generate release body | |
| id: generate-release-body | |
| working-directory: langchain | |
| env: | |
| WORKING_DIR: ${{ inputs.working-directory }} | |
| PKG_NAME: ${{ needs.build.outputs.pkg-name }} | |
| TAG: ${{ steps.check-tags.outputs.tag }} | |
| PREV_TAG: ${{ steps.check-tags.outputs.prev-tag }} | |
| run: | | |
| PREAMBLE="Changes since $PREV_TAG" | |
| # if PREV_TAG is empty, then we are releasing the first version | |
| if [ -z "$PREV_TAG" ]; then | |
| PREAMBLE="Initial release" | |
| PREV_TAG=$(git rev-list --max-parents=0 HEAD) | |
| fi | |
| { | |
| echo 'release-body<<EOF' | |
| echo $PREAMBLE | |
| echo | |
| git log --format="%s" "$PREV_TAG"..HEAD -- $WORKING_DIR | |
| echo EOF | |
| } >> "$GITHUB_OUTPUT" | |
| test-pypi-publish: | |
| needs: | |
| - build | |
| - release-notes | |
| uses: | |
| ./.github/workflows/_test_release.yml | |
| permissions: write-all | |
| with: | |
| working-directory: ${{ inputs.working-directory }} | |
| dangerous-nonmaster-release: ${{ inputs.dangerous-nonmaster-release }} | |
| secrets: inherit | |
| pre-release-checks: | |
| needs: | |
| - build | |
| - release-notes | |
| - test-pypi-publish | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # We explicitly *don't* set up caching here. This ensures our tests are | |
| # maximally sensitive to catching breakage. | |
| # | |
| # For example, here's a way that caching can cause a falsely-passing test: | |
| # - Make the langchain package manifest no longer list a dependency package | |
| # as a requirement. This means it won't be installed by `pip install`, | |
| # and attempting to use it would cause a crash. | |
| # - That dependency used to be required, so it may have been cached. | |
| # When restoring the venv packages from cache, that dependency gets included. | |
| # - Tests pass, because the dependency is present even though it wasn't specified. | |
| # - The package is published, and it breaks on the missing dependency when | |
| # used in the real world. | |
| - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} | |
| uses: "./.github/actions/poetry_setup" | |
| id: setup-python | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| working-directory: ${{ inputs.working-directory }} | |
| - name: Import published package | |
| shell: bash | |
| working-directory: ${{ inputs.working-directory }} | |
| env: | |
| PKG_NAME: ${{ needs.build.outputs.pkg-name }} | |
| VERSION: ${{ needs.build.outputs.version }} | |
| # Here we use: | |
| # - The default regular PyPI index as the *primary* index, meaning | |
| # that it takes priority (https://pypi.org/simple) | |
| # - The test PyPI index as an extra index, so that any dependencies that | |
| # are not found on test PyPI can be resolved and installed anyway. | |
| # (https://test.pypi.org/simple). This will include the PKG_NAME==VERSION | |
| # package because VERSION will not have been uploaded to regular PyPI yet. | |
| # - attempt install again after 5 seconds if it fails because there is | |
| # sometimes a delay in availability on test pypi | |
| run: | | |
| poetry run pip install \ | |
| --extra-index-url https://test.pypi.org/simple/ \ | |
| "$PKG_NAME==$VERSION" || \ | |
| ( \ | |
| sleep 15 && \ | |
| poetry run pip install \ | |
| --extra-index-url https://test.pypi.org/simple/ \ | |
| "$PKG_NAME==$VERSION" \ | |
| ) | |
| # Replace all dashes in the package name with underscores, | |
| # since that's how Python imports packages with dashes in the name. | |
| IMPORT_NAME="$(echo "$PKG_NAME" | sed s/-/_/g)" | |
| poetry run python -c "import $IMPORT_NAME; print(dir($IMPORT_NAME))" | |
| - name: Import test dependencies | |
| run: poetry install --with test | |
| working-directory: ${{ inputs.working-directory }} | |
| # Overwrite the local version of the package with the test PyPI version. | |
| - name: Import published package (again) | |
| working-directory: ${{ inputs.working-directory }} | |
| shell: bash | |
| env: | |
| PKG_NAME: ${{ needs.build.outputs.pkg-name }} | |
| VERSION: ${{ needs.build.outputs.version }} | |
| run: | | |
| poetry run pip install \ | |
| --extra-index-url https://test.pypi.org/simple/ \ | |
| "$PKG_NAME==$VERSION" | |
| - name: Run unit tests | |
| run: make tests | |
| working-directory: ${{ inputs.working-directory }} | |
| - name: Check for prerelease versions | |
| working-directory: ${{ inputs.working-directory }} | |
| run: | | |
| poetry run python $GITHUB_WORKSPACE/.github/scripts/check_prerelease_dependencies.py pyproject.toml | |
| - name: Get minimum versions | |
| working-directory: ${{ inputs.working-directory }} | |
| id: min-version | |
| run: | | |
| poetry run pip install packaging | |
| python_version="$(poetry run python --version | awk '{print $2}')" | |
| min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml release $python_version)" | |
| echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT" | |
| echo "min-versions=$min_versions" | |
| - name: Run unit tests with minimum dependency versions | |
| if: ${{ steps.min-version.outputs.min-versions != '' }} | |
| env: | |
| MIN_VERSIONS: ${{ steps.min-version.outputs.min-versions }} | |
| run: | | |
| poetry run pip install --force-reinstall $MIN_VERSIONS --editable . | |
| make tests | |
| working-directory: ${{ inputs.working-directory }} | |
| - name: 'Authenticate to Google Cloud' | |
| id: 'auth' | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}' | |
| - name: Import integration test dependencies | |
| run: poetry install --with test,test_integration | |
| working-directory: ${{ inputs.working-directory }} | |
| - name: Run integration tests | |
| if: ${{ startsWith(inputs.working-directory, 'libs/partners/') }} | |
| env: | |
| AI21_API_KEY: ${{ secrets.AI21_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} | |
| TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} | |
| AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }} | |
| AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }} | |
| NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} | |
| GOOGLE_SEARCH_API_KEY: ${{ secrets.GOOGLE_SEARCH_API_KEY }} | |
| GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} | |
| EXA_API_KEY: ${{ secrets.EXA_API_KEY }} | |
| NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }} | |
| WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }} | |
| WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} | |
| PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} | |
| PINECONE_ENVIRONMENT: ${{ secrets.PINECONE_ENVIRONMENT }} | |
| ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }} | |
| ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }} | |
| ASTRA_DB_KEYSPACE: ${{ secrets.ASTRA_DB_KEYSPACE }} | |
| ES_URL: ${{ secrets.ES_URL }} | |
| ES_CLOUD_ID: ${{ secrets.ES_CLOUD_ID }} | |
| ES_API_KEY: ${{ secrets.ES_API_KEY }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte | |
| MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }} | |
| VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }} | |
| UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }} | |
| FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} | |
| run: make integration_tests | |
| working-directory: ${{ inputs.working-directory }} | |
| publish: | |
| needs: | |
| - build | |
| - release-notes | |
| - test-pypi-publish | |
| - pre-release-checks | |
| runs-on: ubuntu-latest | |
| permissions: | |
| # This permission is used for trusted publishing: | |
| # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/ | |
| # | |
| # Trusted publishing has to also be configured on PyPI for each package: | |
| # https://docs.pypi.org/trusted-publishers/adding-a-publisher/ | |
| id-token: write | |
| defaults: | |
| run: | |
| working-directory: ${{ inputs.working-directory }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} | |
| uses: "./.github/actions/poetry_setup" | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| working-directory: ${{ inputs.working-directory }} | |
| cache-key: release | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: dist | |
| path: ${{ inputs.working-directory }}/dist/ | |
| - name: Publish package distributions to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: ${{ inputs.working-directory }}/dist/ | |
| verbose: true | |
| print-hash: true | |
| # Temp workaround since attestations are on by default as of gh-action-pypi-publish v1.11.0 | |
| attestations: false | |
| mark-release: | |
| needs: | |
| - build | |
| - release-notes | |
| - test-pypi-publish | |
| - pre-release-checks | |
| - publish | |
| runs-on: ubuntu-latest | |
| permissions: | |
| # This permission is needed by `ncipollo/release-action` to | |
| # create the GitHub release. | |
| contents: write | |
| defaults: | |
| run: | |
| working-directory: ${{ inputs.working-directory }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} | |
| uses: "./.github/actions/poetry_setup" | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| working-directory: ${{ inputs.working-directory }} | |
| cache-key: release | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: dist | |
| path: ${{ inputs.working-directory }}/dist/ | |
| - name: Create Tag | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| artifacts: "dist/*" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| generateReleaseNotes: false | |
| tag: ${{needs.build.outputs.pkg-name}}==${{ needs.build.outputs.version }} | |
| body: ${{ needs.release-notes.outputs.release-body }} | |
| commit: ${{ github.sha }} | |
| makeLatest: ${{ needs.build.outputs.pkg-name == 'langchain-core'}} |