diff

.github/scripts/check_diff.py

@@ -1,15 +1,20 @@
 import json
 import sys
 import os
+from typing import Dict
 
-LANGCHAIN_DIRS = {
-    "libs/genai",
-    "libs/vertexai",
-}
+NVIDIA_DIRS = [
+    "libs/ai-endpoints",
+    "libs/trt",
+]
 
 if __name__ == "__main__":
     files = sys.argv[1:]
-    dirs_to_run = set()
+
+    dirs_to_run: Dict[str, set] = {
+        "lint": set(),
+        "test": set(),
+    }
 
     if len(files) == 300:
         # max diff length is 300 files - there are likely files missing
@@ -25,12 +30,30 @@
                 ".github/scripts/check_diff.py",
             )
         ):
-            dirs_to_run.update(LANGCHAIN_DIRS)
-        elif "libs/genai" in file:
-            dirs_to_run.update({"libs/genai"})
-        elif "libs/vertexai" in file:
-            dirs_to_run.update({"libs/vertexai"})
-        else:
-            pass
-    json_output = json.dumps(list(dirs_to_run))
-    print(f"dirs-to-run={json_output}")
+            # add all LANGCHAIN_DIRS for infra changes
+            dirs_to_run["extended-test"].update(NVIDIA_DIRS)
+            dirs_to_run["lint"].add(".")
+
+        if any(file.startswith(dir_) for dir_ in NVIDIA_DIRS):
+            for dir_ in NVIDIA_DIRS:
+                if file.startswith(dir_):
+                    # add that dir and all dirs after in LANGCHAIN_DIRS
+                    # for extended testing
+                    dirs_to_run["test"].add(dir_)
+        elif file.startswith("libs/"):
+            raise ValueError(
+                f"Unknown lib: {file}. check_diff.py likely needs "
+                "an update for this new library!"
+            )
+        elif any(file.startswith(p) for p in ["docs/", "templates/", "cookbook/"]):
+            dirs_to_run["lint"].add(".")
+
+    outputs = {
+        "dirs-to-lint": list(
+            dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"]
+        ),
+        "dirs-to-test": list(dirs_to_run["test"] | dirs_to_run["extended-test"]),
+    }
+    for key, value in outputs.items():
+        json_output = json.dumps(value)
+        print(f"{key}={json_output}")  # noqa: T201

.github/workflows/check_diffs.yml

@@ -16,6 +16,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  POETRY_VERSION: "1.7.1"
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -30,23 +33,69 @@ jobs:
         run: |
           python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT
     outputs:
-      dirs-to-run: ${{ steps.set-matrix.outputs.dirs-to-run }}
-  codespell:
+      dirs-to-lint: ${{ steps.set-matrix.outputs.dirs-to-lint }}
+      dirs-to-test: ${{ steps.set-matrix.outputs.dirs-to-test }}
+  lint:
+    name: cd ${{ matrix.working-directory }}
+    needs: [ build ]
+    if: ${{ needs.build.outputs.dirs-to-lint != '[]' }}
+    strategy:
+      matrix:
+        working-directory: ${{ fromJson(needs.build.outputs.dirs-to-lint) }}
+    uses: ./.github/workflows/_lint.yml
+    with:
+      working-directory: ${{ matrix.working-directory }}
+    secrets: inherit
+
+  test:
     name: cd ${{ matrix.working-directory }}
-    needs: [build]
+    needs: [ build ]
+    if: ${{ needs.build.outputs.dirs-to-test != '[]' }}
     strategy:
       matrix:
-        working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }}
-    uses: ./.github/workflows/_codespell.yml
+        working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }}
+    uses: ./.github/workflows/_test.yml
     with:
       working-directory: ${{ matrix.working-directory }}
+    secrets: inherit
 
-  ci:
+  compile-integration-tests:
     name: cd ${{ matrix.working-directory }}
     needs: [ build ]
+    if: ${{ needs.build.outputs.dirs-to-test != '[]' }}
     strategy:
       matrix:
-        working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }}
-    uses: ./.github/workflows/_all_ci.yml
+        working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }}
+    uses: ./.github/workflows/_compile_integration_test.yml
     with:
       working-directory: ${{ matrix.working-directory }}
+    secrets: inherit
+
+  dependencies:
+    name: cd ${{ matrix.working-directory }}
+    needs: [ build ]
+    if: ${{ needs.build.outputs.dirs-to-test != '[]' }}
+    strategy:
+      matrix:
+        working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }}
+    uses: ./.github/workflows/_dependencies.yml
+    with:
+      working-directory: ${{ matrix.working-directory }}
+    secrets: inherit
+  ci_success:
+    name: "CI Success"
+    needs: [build, lint, test, compile-integration-tests, dependencies, extended-tests]
+    if: |
+      always()
+    runs-on: ubuntu-latest
+    env:
+      JOBS_JSON: ${{ toJSON(needs) }}
+      RESULTS_JSON: ${{ toJSON(needs.*.result) }}
+      EXIT_CODE: ${{!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && '0' || '1'}}
+    steps:
+      - name: "CI Success"
+        run: |
+          echo $JOBS_JSON
+          echo $RESULTS_JSON
+          echo "Exiting with $EXIT_CODE"
+          exit $EXIT_CODE