diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index f27f7eb4f..000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,586 +0,0 @@ -# We use the "Secure Development Tools" which needs to be installed in any DevOps organization that use -# this YAML file. The free public Azure Pipelines for OSS includes these tools by default already. - -# TODO: add ESLint once it's added to the "Secure Development Tools". TSLint is depricated. - -variables: -# https://github.com/actions/runner-images/tree/main -# TODO: change ARM to use: macOS-13-arm64, and add tests that work on native m1 silicon -- name: python_bld_ver - value: 3.9 -- name: ubuntu_bld - value: ubuntu-20.04 -- name: mac_bld - value: macOS-13 -- name: windows_bld - value: windows-2022 -- name: ubuntu_test - value: ubuntu-latest -- name: mac_test - value: macOS-latest -- name: windows_test - value: windows-latest -- name: skipComponentGovernanceDetection - value: true -- name: skipSCA - value: true -- name: skipNugetSecurityAnalysis - value: true -- name: runCodesignValidationInjection - value: false -- name: Codeql.SkipTaskAutoInjection - value: true - -jobs: -- job: bld - strategy: - matrix: - linux_release_64: - image.name: ${{ variables.ubuntu_bld }} - asm: "-asm" - options: "-release_64" - artifact_name: "libebm_ubuntu_release_64" - linux_debug_64: - image.name: ${{ variables.ubuntu_bld }} - asm: "" - options: "-debug_64" - artifact_name: "libebm_ubuntu_debug_64" - mac_release_64: - image.name: ${{ variables.mac_bld }} - asm: "-asm" - options: "-release_64" - artifact_name: "libebm_mac_release_64" - mac_debug_64: - image.name: ${{ variables.mac_bld }} - asm: "" - options: "-debug_64" - artifact_name: "libebm_mac_debug_64" - mac_release_arm: - image.name: ${{ variables.mac_bld }} - asm: "-asm" - options: "-release_arm" - artifact_name: "libebm_mac_release_arm" - mac_debug_arm: - image.name: ${{ variables.mac_bld }} - asm: "" - options: "-debug_arm" - artifact_name: "libebm_mac_debug_arm" - win_release_64: - image.name: ${{ variables.windows_bld }} - asm: "" - options: "-release_64" - artifact_name: "libebm_win_release_64" - win_debug_64: - image.name: ${{ variables.windows_bld }} - asm: "" - options: "-debug_64" - artifact_name: "libebm_win_debug_64" - maxParallel: 12 - pool: - vmImage: $(image.name) - steps: - - task: PythonScript@0 - inputs: - scriptSource: inline - script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe') - condition: startsWith(variables['image.name'], 'windows') - displayName: CUDA download installer - # CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf - # the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html - - script: | - SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\ - SET CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2 - cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2 - IF ERRORLEVEL 1 ( - ECHO cuda_11.2.2_win10_network.exe FAILED - EXIT /B 201 - ) - .\build.bat $(options) $(asm) - condition: startsWith(variables['image.name'], 'windows') - displayName: win bld_native - - script: | - sudo apt --yes update - if [ $? -ne 0 ]; then - exit 107 - fi - sudo apt --yes install nvidia-cuda-toolkit - if [ $? -ne 0 ]; then - exit 93 - fi - /bin/sh ./build.sh $(options) $(asm) - condition: startsWith(variables['image.name'], 'ubuntu') - displayName: linux bld_native - - script: | - /bin/sh ./build.sh $(options) $(asm) - condition: startsWith(variables['image.name'], 'macOS') - displayName: mac bld_native - - publish: bld/lib - artifact: $(artifact_name) - displayName: Publish native shared library - - publish: bld/asm/ - artifact: asm-$(artifact_name) - condition: ne(variables['asm'], '') - displayName: Publish assembly x64 - -- job: vis - pool: - vmImage: ${{ variables.ubuntu_bld }} - steps: - - script: | - cd shared/vis - npm install - npm run build-prod - displayName: bld_vis - - publish: shared/vis/dist - artifact: vis - displayName: Publish interpret-inline.js library - -- job: npm - pool: - vmImage: ${{ variables.ubuntu_bld }} - steps: - - script: | - cd shared/vis - npm install - npm run build-prod - npm pack - mkdir pkg - cp *.tgz pkg/ - displayName: bld_npm_package - - publish: shared/vis/pkg - artifact: npm - displayName: Publish npm package - -- job: R - pool: - vmImage: ${{ variables.ubuntu_bld }} - steps: - - script: | - cd R - sudo apt --yes update - if [ $? -ne 0 ]; then - exit 201 - fi - sudo apt --yes install texlive-latex-base texlive-fonts-extra - if [ $? -ne 0 ]; then - exit 201 - fi - Rscript build.R - displayName: Rscript build.R - - script: cat bld/tmp/R/interpret.Rcheck/00install.out - condition: failed() - displayName: Display errors - - publish: bld/R - artifact: R - displayName: Publish R package - -- job: sdist - pool: - vmImage: ${{ variables.ubuntu_bld }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ variables.python_bld_ver }} - - script: | - python -m pip install --upgrade pip setuptools wheel - cd python/interpret-core - python setup.py sdist -d ../../bld/sdist - cd ../interpret - python setup.py sdist -d ../../bld/sdist - displayName: interpret bld_sdist_package - - publish: bld/sdist - artifact: sdist - displayName: Publish sdist python package - -- job: docs - dependsOn: sdist - pool: - vmImage: ${{ variables.ubuntu_bld }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ variables.python_bld_ver }} - - task: DownloadPipelineArtifact@2 - inputs: - artifact: sdist - itemPattern: interpret_core-*.tar.gz - path: "$(System.ArtifactsDirectory)/sdist" - displayName: Download sdist - - script: | - cd "$(System.ArtifactsDirectory)/sdist" - tarball_path=$(echo interpret_core-*.tar.gz) - python -m pip install --upgrade pip setuptools wheel - python -m pip install "$tarball_path"[debug,notebook,plotly,lime,sensitivity,shap,linear,treeinterpreter,aplr,dash,skoperules,testing] - cd "$(Build.SourcesDirectory)/docs/" - pip install -r requirements.txt - /bin/sh ./build.sh - continueOnError: true - displayName: Build docs - - publish: docs/interpret/_build/html/ - artifact: docs - displayName: Publish docs - -- job: bdist - dependsOn: [bld, vis] - pool: - vmImage: ${{ variables.ubuntu_bld }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ variables.python_bld_ver }} - - task: DownloadPipelineArtifact@2 - inputs: - itemPattern: | - **/libebm*.dll - **/libebm*.pdb - **/libebm*.so - **/libebm*.dylib - **/interpret-inline.js - **/interpret-inline.js.LICENSE.txt - path: "$(System.ArtifactsDirectory)/lib/" - displayName: Download artifact lib files - - task: CopyFiles@2 - inputs: - SourceFolder: "$(System.ArtifactsDirectory)/lib/" - TargetFolder: python/interpret-core/interpret/root/bld/lib/ - flattenFolders: true - displayName: Flatten and copy lib files - - script: | - python -m pip install --upgrade pip setuptools wheel - cd python/interpret-core - python setup.py bdist_wheel -d ../../bld/bdist - cd ../interpret - python setup.py bdist_wheel -d ../../bld/bdist - displayName: interpret bld_bdist_package - - publish: bld/bdist - artifact: bdist - displayName: Publish bdist python package - -- job: testC - dependsOn: bld - strategy: - matrix: - linux_release_64: - image.name: ${{ variables.ubuntu_test }} - options: "-release_64 -existing_release_64" - scheduled: "-valgrind" - artifact_name: "libebm_ubuntu_release_64" - linux_debug_64: - image.name: ${{ variables.ubuntu_test }} - options: "-debug_64 -existing_debug_64" - scheduled: "-valgrind" - artifact_name: "libebm_ubuntu_debug_64" - linux_release_32: - image.name: ${{ variables.ubuntu_test }} - options: "-release_32" - scheduled: "-valgrind" - artifact_name: "" - linux_debug_32: - image.name: ${{ variables.ubuntu_test }} - options: "-debug_32" - scheduled: "-valgrind" - artifact_name: "" - mac_release_64: - image.name: ${{ variables.mac_test }} - options: "-release_64 -existing_release_64" - scheduled: "" - artifact_name: "libebm_mac_release_64" - mac_debug_64: - image.name: ${{ variables.mac_test }} - # don't use the existing debug library since we want to rebuild with asan - options: "-debug_64 -asan" - scheduled: "" - artifact_name: "" - win_release_64: - image.name: ${{ variables.windows_test }} - options: "-release_64 -existing_release_64" - scheduled: "-analysis" - artifact_name: "libebm_win_release_64" - win_debug_64: - image.name: ${{ variables.windows_test }} - options: "-debug_64 -existing_debug_64" - scheduled: "" - artifact_name: "libebm_win_debug_64" - win_release_32: - image.name: ${{ variables.windows_test }} - options: "-release_32" - scheduled: "-analysis" - artifact_name: "" - win_debug_32: - image.name: ${{ variables.windows_test }} - options: "-debug_32" - scheduled: "" - artifact_name: "" - maxParallel: 10 - pool: - vmImage: $(image.name) - steps: - - task: DownloadPipelineArtifact@2 - inputs: - artifact: $(artifact_name) - path: bld/lib - condition: ne(variables['artifact_name'], '') - displayName: Copy native library to bld/lib directory - - task: PythonScript@0 - inputs: - scriptSource: inline - script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe') - condition: startsWith(variables['image.name'], 'windows') - displayName: Download CUDA installer - - script: | - /bin/sh ./shared/libebm/tests/libebm_test.sh $(options) - condition: startsWith(variables['image.name'], 'macOS') - displayName: mac test_native - - script: | - /bin/sh ./shared/libebm/tests/libebm_test.sh $(options) - condition: and(startsWith(variables['image.name'], 'ubuntu'), ne(variables['Build.Reason'], 'Schedule')) - displayName: linux test_native (CI) - - script: | - /bin/sh ./shared/libebm/tests/libebm_test.sh $(options) $(scheduled) - condition: and(startsWith(variables['image.name'], 'ubuntu'), eq(variables['Build.Reason'], 'Schedule')) - displayName: linux test_native (Schedule) - - script: | - SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\ - SET CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2 - cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2 - IF ERRORLEVEL 1 ( - ECHO cuda_11.2.2_win10_network.exe FAILED - EXIT /B 201 - ) - .\shared\libebm\tests\libebm_test.bat $(options) - condition: and(startsWith(variables['image.name'], 'windows'), ne(variables['Build.Reason'], 'Schedule')) - displayName: win test_native (CI) - - script: | - SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\ - SET CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2 - cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2 - IF ERRORLEVEL 1 ( - ECHO cuda_11.2.2_win10_network.exe FAILED - EXIT /B 201 - ) - .\shared\libebm\tests\libebm_test.bat $(options) $(scheduled) - condition: and(startsWith(variables['image.name'], 'windows'), eq(variables['Build.Reason'], 'Schedule')) - displayName: win test_native (Schedule) - -- job: testS - dependsOn: sdist - strategy: - matrix: - linux_39_python: - python.version: 3.9 - image.name: ${{ variables.ubuntu_test }} - linux_310_python: - python.version: 3.10 - image.name: ${{ variables.ubuntu_test }} - linux_311_python: - python.version: 3.11 - image.name: ${{ variables.ubuntu_test }} - linux_312_python: - python.version: 3.12 - image.name: ${{ variables.ubuntu_test }} - win_39_python: - python.version: 3.9 - image.name: ${{ variables.windows_test }} - win_310_python: - python.version: 3.10 - image.name: ${{ variables.windows_test }} - win_311_python: - python.version: 3.11 - image.name: ${{ variables.windows_test }} - win_312_python: - python.version: 3.12 - image.name: ${{ variables.windows_test }} - mac_39_python: - python.version: 3.9 - image.name: ${{ variables.mac_test }} - mac_310_python: - python.version: 3.10 - image.name: ${{ variables.mac_test }} - mac_311_python: - python.version: 3.11 - image.name: ${{ variables.mac_test }} - mac_312_python: - python.version: 3.12 - image.name: ${{ variables.mac_test }} - maxParallel: 9 - pool: - vmImage: $(image.name) - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: $(python.version) - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: sdist - itemPattern: interpret_core-*.tar.gz - targetPath: "$(System.ArtifactsDirectory)/sdist" - - script: | - python -m pip install --upgrade pip setuptools wheel - FOR %%A IN ("$(System.ArtifactsDirectory)\sdist\interpret_core-*.tar.gz") DO SET "install_file=%%~nA" - REM the above line got "interpret_core-x.x.x.tar" and we now need to strip the ".tar" - FOR %%A IN ("%install_file%") DO SET "install_file=%%~nA" - SET PATH=%PATH%;C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\ - python -m pip install "$(System.ArtifactsDirectory)\sdist\%install_file%.tar.gz"[debug,notebook,plotly,lime,sensitivity,shap,linear,treeinterpreter,aplr,dash,skoperules,testing] - mkdir "$(Agent.TempDirectory)\zqmr" - mkdir "$(Agent.TempDirectory)\zqmr\t" - tar -xzvf "$(System.ArtifactsDirectory)\sdist\%install_file%.tar.gz" -C "$(Agent.TempDirectory)\zqmr\t" "%install_file%/tests" - cd /D "$(Agent.TempDirectory)\zqmr\t" - set PATH=%PATH%;%GeckoWebDriver% - python -m pytest -vv -n auto --junitxml=junit/test-results.xml --cov=interpret --cov-report=xml --cov-report=html - condition: startsWith(variables['image.name'], 'windows') - displayName: pytest (win) - - script: | - python -m pip install --upgrade pip setuptools wheel - cd "$(System.ArtifactsDirectory)/sdist" - install_file=$(echo interpret_core-*.tar.gz) - install_file=${install_file%.tar.gz} - python -m pip install "$(System.ArtifactsDirectory)/sdist/$install_file.tar.gz"[debug,notebook,plotly,lime,sensitivity,shap,linear,treeinterpreter,aplr,dash,skoperules,testing] - mkdir -p "$(Agent.TempDirectory)/zqmr/t" - tar -xzvf "$(System.ArtifactsDirectory)/sdist/$install_file.tar.gz" -C "$(Agent.TempDirectory)/zqmr/t" "$install_file/tests" - cd "$(Agent.TempDirectory)/zqmr/t" - python -m pytest -vv -n auto --junitxml=junit/test-results.xml --cov=interpret --cov-report=xml --cov-report=html - condition: not(startsWith(variables['image.name'], 'windows')) - displayName: pytest (non-win) - - task: PublishTestResults@2 - inputs: - testResultsFiles: "$(Agent.TempDirectory)/zqmr/t/junit/test-results.xml" - testRunTitle: Publish test results for Python sdist $(python.version) at $(image.name) - condition: succeededOrFailed() - displayName: Publish test results - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: "$(Agent.TempDirectory)/zqmr/t/coverage.xml" - # reportDirectory: $(Agent.TempDirectory)/zqmr/t/**/htmlcov - condition: startsWith(variables['image.name'], 'windows') - displayName: Publish test coverage results - -- job: testB - dependsOn: bdist - strategy: - matrix: - linux_39_python: - python.version: 3.9 - image.name: ${{ variables.ubuntu_test }} - linux_310_python: - python.version: 3.10 - image.name: ${{ variables.ubuntu_test }} - linux_311_python: - python.version: 3.11 - image.name: ${{ variables.ubuntu_test }} - linux_312_python: - python.version: 3.12 - image.name: ${{ variables.ubuntu_test }} - win_39_python: - python.version: 3.9 - image.name: ${{ variables.windows_test }} - win_310_python: - python.version: 3.10 - image.name: ${{ variables.windows_test }} - win_311_python: - python.version: 3.11 - image.name: ${{ variables.windows_test }} - win_312_python: - python.version: 3.12 - image.name: ${{ variables.windows_test }} - mac_39_python: - python.version: 3.9 - image.name: ${{ variables.mac_test }} - mac_310_python: - python.version: 3.10 - image.name: ${{ variables.mac_test }} - mac_311_python: - python.version: 3.11 - image.name: ${{ variables.mac_test }} - mac_312_python: - python.version: 3.12 - image.name: ${{ variables.mac_test }} - maxParallel: 9 - pool: - vmImage: $(image.name) - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: $(python.version) - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: bdist - itemPattern: interpret_core-*.whl - targetPath: "$(System.ArtifactsDirectory)/bdist" - # we copy the tests to a new location to test the installed package rather than the source files - - script: | - python -m pip install --upgrade pip setuptools wheel - FOR %%A IN ("$(System.ArtifactsDirectory)\bdist\interpret_core-*.whl") DO SET "install_file=%%A" - python -m pip install "%install_file%"[debug,notebook,plotly,lime,sensitivity,shap,linear,treeinterpreter,aplr,dash,skoperules,testing] - set PATH=%PATH%;%GeckoWebDriver% - mkdir "$(Agent.TempDirectory)\zqmr" - mkdir "$(Agent.TempDirectory)\zqmr\t" - xcopy /E "$(Build.SourcesDirectory)\python\interpret-core\tests\*" "$(Agent.TempDirectory)\zqmr\t\" - cd /D "$(Agent.TempDirectory)\zqmr\t" - python -m pytest -vv -n auto --junitxml=junit/test-results.xml --cov=interpret --cov-report=xml --cov-report=html - condition: startsWith(variables['image.name'], 'windows') - displayName: pytest (win) - # we copy the tests to a new location to test the installed package rather than the source files - - script: | - python -m pip install --upgrade pip setuptools wheel - cd "$(System.ArtifactsDirectory)/bdist" - install_file=$(echo interpret_core-*.whl) - python -m pip install "$install_file"[debug,notebook,plotly,lime,sensitivity,shap,linear,treeinterpreter,aplr,dash,skoperules,testing] - mkdir -p "$(Agent.TempDirectory)/zqmr/t" - cp -r "$(Build.SourcesDirectory)/python/interpret-core/tests/" "$(Agent.TempDirectory)/zqmr/t/" - cd "$(Agent.TempDirectory)/zqmr/t" - python -m pytest -vv -n auto --junitxml=junit/test-results.xml --cov=interpret --cov-report=xml --cov-report=html - condition: not(startsWith(variables['image.name'], 'windows')) - displayName: pytest (non-win) - - task: PublishTestResults@2 - inputs: - testResultsFiles: "$(Agent.TempDirectory)/zqmr/t/junit/test-results.xml" - testRunTitle: Publish test results for Python bdist $(python.version) at $(image.name) - condition: succeededOrFailed() - displayName: Publish test results - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: "$(Agent.TempDirectory)/zqmr/t/coverage.xml" - # reportDirectory: $(Agent.TempDirectory)/zqmr/t/**/htmlcov - condition: startsWith(variables['image.name'], 'windows') - displayName: Publish test coverage results - -- job: clang_format - dependsOn: [testC, testS, testB] - condition: always() - pool: - vmImage: ${{ variables.ubuntu_bld }} - steps: - - script: | - echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-jammy-16.list - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg - sudo apt --yes update - sudo apt --yes install clang-format-16 - find shared/libebm \( -iname "*.cpp" -o -iname "*.h" -o -iname "*.hpp" \) | xargs clang-format-16 -i -style=file - git diff --exit-code - continueOnError: true - displayName: Check C++ code formatting with clang-format - -- job: format_black - dependsOn: [testC, testS, testB] - condition: always() - pool: - vmImage: ${{ variables.ubuntu_bld }} - steps: - - script: | - cd python/interpret-core - python -m pip install --upgrade black - black --check . - continueOnError: true - displayName: black python formatting check - - publish: "$(Build.SourcesDirectory)" - artifact: source_code - -schedules: -- cron: "0 12 * * *" - branches: - include: - - develop - always: true - displayName: Daily midnight build