Merge pull request #45 from jmolina4/local_over_docker #9
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Local Setup Test | |
| on: [push] | |
| jobs: | |
| mac_nix: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: "zulu" | |
| java-version: "11" | |
| - name: Install poetry | |
| run: pipx install poetry | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: "poetry" | |
| - name: Install Python Dependencies | |
| run: | | |
| poetry install | |
| - name: Run local unit tests | |
| run: | | |
| ./go.sh run-local-unit-test | |
| - name: Run local integration tests | |
| run: | | |
| ./go.sh run-local-integration-test | |
| windows: | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: "zulu" | |
| java-version: "11" | |
| - name: Install poetry | |
| run: pipx install poetry | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: "poetry" | |
| - name: Install Hadoop for Windows | |
| # See https://cwiki.apache.org/confluence/display/HADOOP2/WindowsProblems | |
| # that recommends https://github.com/steveloughran/winutils | |
| # that recommends https://github.com/cdarlint/winutils | |
| # | |
| # Setting environement variables: e.g. | |
| # $env:HADOOP_HOME = "$pwd\winutils\hadoop-3.3.5" | |
| # $env:Path += ";$pwd\winutils\hadoop-3.3.5\bin" | |
| # requires a special handling: https://stackoverflow.com/questions/61858388/how-do-i-set-an-enviroment-variable-in-github-action-on-a-windows-server | |
| # | |
| # Reading / Writing to parquet through winutils requires Microsoft Visual C++ 2010 Service Pack 1 | |
| # https://stackoverflow.com/questions/45947375/why-does-starting-a-streaming-query-lead-to-exitcodeexception-exitcode-1073741 | |
| run: | | |
| choco install vcredist2010 | |
| git clone --depth 1 -b master https://github.com/cdarlint/winutils.git | |
| echo "HADOOP_HOME=$pwd\winutils\hadoop-3.3.5" >> $env:GITHUB_ENV | |
| echo ";$pwd\winutils\hadoop-3.3.5\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| - name: Test Hadoop Setup | |
| run: | | |
| winutils.exe chmod 777 D:\a\dataengineer-transformations-python\dataengineer-transformations-python | |
| - name: Install Python Dependencies | |
| run: | | |
| scripts\install.bat | |
| poetry install | |
| - name: Run local unit tests | |
| run: | | |
| .\go.ps1 run-local-unit-test | |
| - name: Run local integration tests | |
| run: | | |
| .\go.ps1 run-local-integration-test |