diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c37f8f2a..a2ac53641 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,13 @@ jobs: codejail_ci: name: tests runs-on: ubuntu-20.04 + strategy: + matrix: + include: + - python_version: '3.8' + docker_tag: latest + - python_version: '3.11' + docker_tag: '3.11' steps: - uses: actions/checkout@v2 @@ -24,15 +31,16 @@ jobs: uses: aws-actions/amazon-ecr-login@v1 - name: Parse custom apparmor profile - run: sudo apparmor_parser -r -W apparmor-profiles/home.sandbox.codejail_sandbox-python3.8.bin.python - + run: sudo apparmor_parser -r -W apparmor-profiles/home.sandbox.codejail_sandbox-python3.bin.python + - name: Pull codejail CI image run: docker pull 257477529851.dkr.ecr.us-east-1.amazonaws.com/openedx-codejail:latest - name: Build latest code changes into CI image run: | docker build --cache-from 257477529851.dkr.ecr.us-east-1.amazonaws.com/openedx-codejail \ - -t 257477529851.dkr.ecr.us-east-1.amazonaws.com/openedx-codejail . + -t 257477529851.dkr.ecr.us-east-1.amazonaws.com/openedx-codejail \ + --build-arg python_version=${{ matrix.python_version }} . - name: Run container with custom apparmor profile and codejail CI image run: | @@ -41,7 +49,7 @@ jobs: - name: Run Non Proxy Tests run: docker exec -t codejail bash -c 'make clean && make test_no_proxy' - + - name: Run Proxy Tests run: docker exec -t codejail bash -c 'make clean && make test_proxy' diff --git a/.github/workflows/push-docker-image.yml b/.github/workflows/push-docker-image.yml index 3eb8feb21..121ae9ac2 100644 --- a/.github/workflows/push-docker-image.yml +++ b/.github/workflows/push-docker-image.yml @@ -8,6 +8,13 @@ on: jobs: push: runs-on: ubuntu-latest + strategy: + matrix: + include: + - python_version: '3.8' + docker_tag: latest + - python_version: '3.11' + docker_tag: '3.11' steps: - name: Checkout @@ -28,7 +35,6 @@ jobs: env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: openedx-codejail - IMAGE_TAG: latest run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:${{ matrix.docker_tag }} --build-arg ${{ matrix.python_version }} . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ matrix.docker_tag }} diff --git a/Dockerfile b/Dockerfile index e74c91cba..18e74c8b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,28 @@ FROM ubuntu:focal SHELL ["/bin/bash", "-c"] +ARG python_version=3.8 + # Install Codejail Packages -RUN apt-get update && apt-get upgrade -y -RUN apt-get install -y vim python3-virtualenv python3-pip -RUN apt-get install -y sudo git +ENV TZ=Etc/UTC +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:deadsnakes/ppa && apt-get update && apt-get upgrade -y +RUN apt-get install -y vim python${python_version} python${python_version}-dev python${python_version}-distutils +RUN apt-get install -y sudo git make curl build-essential +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${python_version} +RUN pip install virtualenv # Define Environment Variables ENV CODEJAIL_GROUP=sandbox ENV CODEJAIL_SANDBOX_CALLER=ubuntu ENV CODEJAIL_TEST_USER=sandbox -ENV CODEJAIL_TEST_VENV=/home/sandbox/codejail_sandbox-python3.8 +ENV CODEJAIL_TEST_VENV=/home/sandbox/codejail_sandbox-python${python_version} # Create Virtualenv for sandbox user -RUN virtualenv -p python3.8 --always-copy $CODEJAIL_TEST_VENV +RUN virtualenv -p python${python_version} --always-copy $CODEJAIL_TEST_VENV -RUN virtualenv -p python3.8 venv +RUN virtualenv -p python${python_version} venv ENV VIRTUAL_ENV=/venv # Add venv/bin to path @@ -48,7 +55,7 @@ RUN pip install -r /codejail/requirements/sandbox.txt && pip install -r /codejai COPY . /codejail # Setup sudoers file -COPY sudoers-file/01-sandbox /etc/sudoers.d/01-sandbox +COPY sudoers-file/01-sandbox-python-${python_version} /etc/sudoers.d/01-sandbox # Change Sudoers file permissions RUN chmod 0440 /etc/sudoers.d/01-sandbox diff --git a/apparmor-profiles/home.sandbox.codejail_sandbox-python3.8.bin.python b/apparmor-profiles/home.sandbox.codejail_sandbox-python3.8.bin.python deleted file mode 100644 index 57d98bd27..000000000 --- a/apparmor-profiles/home.sandbox.codejail_sandbox-python3.8.bin.python +++ /dev/null @@ -1,26 +0,0 @@ -#include - -profile apparmor_profile /home/sandbox/codejail_sandbox-python3.8/bin/python { - #include - #include - - /home/sandbox/codejail_sandbox-python3.8/** mr, - /tmp/codejail-*/ rix, - /tmp/codejail-*/** wrix, - - # Whitelist particiclar shared objects from the system - # python installation - # - /usr/lib/python3.8/lib-dynload/_json.so mr, - /usr/lib/python3.8/lib-dynload/_ctypes.so mr, - /usr/lib/python3.8/lib-dynload/_heapq.so mr, - /usr/lib/python3.8/lib-dynload/_io.so mr, - /usr/lib/python3.8/lib-dynload/_csv.so mr, - /usr/lib/python3.8/lib-dynload/datetime.so mr, - /usr/lib/python3.8/lib-dynload/_elementtree.so mr, - /usr/lib/python3.8/lib-dynload/pyexpat.so mr, - # - # Allow access to selections from /proc - # - /proc/*/mounts r, -} diff --git a/apparmor-profiles/home.sandbox.codejail_sandbox-python3.bin.python b/apparmor-profiles/home.sandbox.codejail_sandbox-python3.bin.python new file mode 100644 index 000000000..3bd38c5c2 --- /dev/null +++ b/apparmor-profiles/home.sandbox.codejail_sandbox-python3.bin.python @@ -0,0 +1,54 @@ +#include + +profile apparmor_profile /home/sandbox/codejail_sandbox-python{3.[0-9],3.[1-9][0-9]}/bin/python { + #include + #include + + /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/**.{pyc,so,so.*[0-9]} mr, + /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/**.{egg,py,pth} r, + /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/ r, + /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/**/ r, + /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/*.dist-info/{METADATA,namespace_packages.txt} r, + /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/*.VERSION r, + /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/*.egg-info/PKG-INFO r, + /usr/{local/,}lib{,32,64}/python3.{1,}[0-9]/lib-dynload/*.so mr, + + # Site-wide configuration + /etc/python{2.[4-7],3.[0-9],3.[1-9][0-9]}/** r, + + # shared python paths + /usr/share/{pyshared,pycentral,python-support}/** r, + /{var,usr}/lib/{pyshared,pycentral,python-support}/** r, + /usr/lib/{pyshared,pycentral,python-support}/**.so mr, + /var/lib/{pyshared,pycentral,python-support}/**.pyc mr, + /usr/lib/python3/dist-packages/**.so mr, + + # wx paths + /usr/lib/wx/python/*.pth r, + + # python build configuration and headers + /usr/include/python{2.[4-7],3.[0-9],3.[1-9][0-9]}*/pyconfig.h r, + + # Include additions to the abstraction + include if exists + + /home/sandbox/codejail_sandbox-python{3.[0-9],3.[1-9][0-9]}/** mr, + /tmp/codejail-*/ rix, + /tmp/codejail-*/** wrix, + + # Whitelist particiclar shared objects from the system + # python installation + # + /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_json.so mr, + /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_ctypes.so mr, + /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_heapq.so mr, + /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_io.so mr, + /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_csv.so mr, + /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/datetime.so mr, + /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_elementtree.so mr, + /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/pyexpat.so mr, + # + # Allow access to selections from /proc + # + /proc/*/mounts r, +} diff --git a/codejail/__init__.py b/codejail/__init__.py index 4bdb01ff5..0420a3745 100644 --- a/codejail/__init__.py +++ b/codejail/__init__.py @@ -1,3 +1,3 @@ """init""" -__version__ = '3.3.3' +__version__ = '3.4.0' diff --git a/codejail/tests/test_jail_code.py b/codejail/tests/test_jail_code.py index 104d57a1c..f11fde9ed 100644 --- a/codejail/tests/test_jail_code.py +++ b/codejail/tests/test_jail_code.py @@ -89,12 +89,14 @@ def test_ends_with_exception(self): res = jailpy(code="""raise Exception('FAIL')""") self.assertNotEqual(res.status, 0) self.assertEqual(res.stdout, b"") - self.assertEqual(res.stderr, bytes(textwrap.dedent("""\ - Traceback (most recent call last): - File "jailed_code", line 1, in - raise Exception('FAIL') + + regex = textwrap.dedent(""" + (?m)Traceback [(]most recent call last[)]: + File "(/.*?/)?jailed_code", line 1, in + raise Exception[(]'FAIL'[)] Exception: FAIL - """), 'utf-8')) + """).strip() + "\n" + self.assertRegex(res.stderr.decode('utf-8'), regex) def test_stdin_is_provided(self): res = jailpy( diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index cca3ccd30..e3bf8eaec 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -13,13 +13,20 @@ # using LTS django version -Django<4.0 +Django<5.0 # elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html elasticsearch<7.14.0 -setuptools<60 - # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected django-simple-history==3.0.0 + +# opentelemetry requires version 6.x at the moment: +# https://github.com/open-telemetry/opentelemetry-python/issues/3570 +# Normally this could be added as a constraint in edx-django-utils, where we're +# adding the opentelemetry dependency. However, when we compile pip-tools.txt, +# that uses version 7.x, and then there's no undoing that when compiling base.txt. +# So we need to pin it globally, for now. +# Ticket for unpinning: https://github.com/openedx/edx-lint/issues/407 +importlib-metadata<7 diff --git a/requirements/development.txt b/requirements/development.txt index b55e7ca98..e0ab4916f 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,28 +1,21 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -astroid==2.11.6 # via -r requirements/testing.txt, pylint -attrs==21.4.0 # via -r requirements/testing.txt, pytest -dill==0.3.5.1 # via -r requirements/testing.txt, pylint -iniconfig==1.1.1 # via -r requirements/testing.txt, pytest -isort==5.10.1 # via -r requirements/testing.txt, pylint -lazy-object-proxy==1.7.1 # via -r requirements/testing.txt, astroid +astroid==3.1.0 # via -r requirements/testing.txt, pylint +dill==0.3.8 # via -r requirements/testing.txt, pylint +exceptiongroup==1.2.1 # via -r requirements/testing.txt, pytest +iniconfig==2.0.0 # via -r requirements/testing.txt, pytest +isort==5.13.2 # via -r requirements/testing.txt, pylint mccabe==0.7.0 # via -r requirements/testing.txt, pylint -packaging==21.3 # via -r requirements/testing.txt, pytest -platformdirs==2.5.2 # via -r requirements/testing.txt, pylint -pluggy==1.0.0 # via -r requirements/testing.txt, pytest -py==1.11.0 # via -r requirements/testing.txt, pytest -pycodestyle==2.8.0 # via -r requirements/testing.txt -pylint==2.14.3 # via -r requirements/testing.txt -pyparsing==3.0.9 # via -r requirements/testing.txt, packaging -pytest==7.1.2 # via -r requirements/testing.txt +packaging==24.0 # via -r requirements/testing.txt, pytest +platformdirs==4.2.1 # via -r requirements/testing.txt, pylint +pluggy==1.5.0 # via -r requirements/testing.txt, pytest +pycodestyle==2.11.1 # via -r requirements/testing.txt +pylint==3.1.0 # via -r requirements/testing.txt +pytest==8.1.1 # via -r requirements/testing.txt tomli==2.0.1 # via -r requirements/testing.txt, pylint, pytest -tomlkit==0.11.0 # via -r requirements/testing.txt, pylint -typing-extensions==4.2.0 # via -r requirements/testing.txt, astroid, pylint -wrapt==1.14.1 # via -r requirements/testing.txt, astroid - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +tomlkit==0.12.4 # via -r requirements/testing.txt, pylint +typing-extensions==4.11.0 # via -r requirements/testing.txt, astroid, pylint diff --git a/requirements/pip.txt b/requirements/pip.txt index 288ee524f..8eb91a436 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,11 +1,11 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -wheel==0.37.1 # via -r requirements/pip.in +wheel==0.43.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==22.1.2 # via -r requirements/pip.in -setuptools==59.8.0 # via -c requirements/common_constraints.txt, -r requirements/pip.in +pip==24.0 # via -r requirements/pip.in +setuptools==69.5.1 # via -r requirements/pip.in diff --git a/requirements/pip_tools.txt b/requirements/pip_tools.txt index 8c7f6dad0..eeeabeeb7 100644 --- a/requirements/pip_tools.txt +++ b/requirements/pip_tools.txt @@ -1,14 +1,18 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -click==8.1.3 # via pip-tools -pep517==0.12.0 # via pip-tools -pip-tools==7.3.0 # via -r requirements/pip_tools.in -tomli==2.0.1 # via pep517 -wheel==0.37.1 # via pip-tools +build==1.2.1 # via pip-tools +click==8.1.7 # via pip-tools +importlib-metadata==6.11.0 # via -c requirements/common_constraints.txt, build +packaging==24.0 # via build +pip-tools==7.4.1 # via -r requirements/pip_tools.in +pyproject-hooks==1.0.0 # via build, pip-tools +tomli==2.0.1 # via build, pip-tools, pyproject-hooks +wheel==0.43.0 # via pip-tools +zipp==3.18.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/sandbox.txt b/requirements/sandbox.txt index fd7d60a87..4aeb57e13 100644 --- a/requirements/sandbox.txt +++ b/requirements/sandbox.txt @@ -1,12 +1,13 @@ # # This file is autogenerated by pip-compile with Python 3.8 -# To update, run: +# by the following command: # # make upgrade # -asgiref==3.5.2 # via django -django==3.2.17 # via -c requirements/common_constraints.txt, -c requirements/constraints.txt, -r requirements/sandbox.in -numpy==1.22.4 # via -r requirements/sandbox.in -pytz==2022.1 # via django +asgiref==3.8.1 # via django +django==3.2.25 # via -c requirements/common_constraints.txt, -c requirements/constraints.txt, -r requirements/sandbox.in +numpy==1.24.4 # via -r requirements/sandbox.in +pytz==2024.1 # via django six==1.16.0 # via -r requirements/sandbox.in -sqlparse==0.4.2 # via django +sqlparse==0.5.0 # via django +typing-extensions==4.11.0 # via asgiref diff --git a/requirements/testing.txt b/requirements/testing.txt index 95ea106d2..cb21847e0 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,28 +1,21 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -astroid==2.11.6 # via pylint -attrs==21.4.0 # via pytest -dill==0.3.5.1 # via pylint -iniconfig==1.1.1 # via pytest -isort==5.10.1 # via -r requirements/testing.in, pylint -lazy-object-proxy==1.7.1 # via astroid +astroid==3.1.0 # via pylint +dill==0.3.8 # via pylint +exceptiongroup==1.2.1 # via pytest +iniconfig==2.0.0 # via pytest +isort==5.13.2 # via -r requirements/testing.in, pylint mccabe==0.7.0 # via pylint -packaging==21.3 # via pytest -platformdirs==2.5.2 # via pylint -pluggy==1.0.0 # via pytest -py==1.11.0 # via pytest -pycodestyle==2.8.0 # via -r requirements/testing.in -pylint==2.14.3 # via -r requirements/testing.in -pyparsing==3.0.9 # via packaging -pytest==7.1.2 # via -r requirements/testing.in +packaging==24.0 # via pytest +platformdirs==4.2.1 # via pylint +pluggy==1.5.0 # via pytest +pycodestyle==2.11.1 # via -r requirements/testing.in +pylint==3.1.0 # via -r requirements/testing.in +pytest==8.1.1 # via -r requirements/testing.in tomli==2.0.1 # via pylint, pytest -tomlkit==0.11.0 # via pylint -typing-extensions==4.2.0 # via astroid, pylint -wrapt==1.14.1 # via astroid - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +tomlkit==0.12.4 # via pylint +typing-extensions==4.11.0 # via astroid, pylint diff --git a/requirements/tox.txt b/requirements/tox.txt index ae38e0e36..ac00fa54a 100644 --- a/requirements/tox.txt +++ b/requirements/tox.txt @@ -1,17 +1,18 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -distlib==0.3.4 # via virtualenv -filelock==3.7.1 # via tox, virtualenv -packaging==21.3 # via tox -platformdirs==2.5.2 # via virtualenv -pluggy==1.0.0 # via tox -py==1.11.0 # via tox -pyparsing==3.0.9 # via packaging -six==1.16.0 # via tox, virtualenv -toml==0.10.2 # via tox -tox==3.25.0 # via -r requirements/tox.in -virtualenv==20.14.1 # via tox +cachetools==5.3.3 # via tox +chardet==5.2.0 # via tox +colorama==0.4.6 # via tox +distlib==0.3.8 # via virtualenv +filelock==3.13.4 # via tox, virtualenv +packaging==24.0 # via pyproject-api, tox +platformdirs==4.2.1 # via tox, virtualenv +pluggy==1.5.0 # via tox +pyproject-api==1.6.1 # via tox +tomli==2.0.1 # via pyproject-api, tox +tox==4.14.2 # via -r requirements/tox.in +virtualenv==20.26.0 # via tox diff --git a/sudoers-file/01-sandbox-python-3.11 b/sudoers-file/01-sandbox-python-3.11 new file mode 100644 index 000000000..46c3cb2a7 --- /dev/null +++ b/sudoers-file/01-sandbox-python-3.11 @@ -0,0 +1,7 @@ +ubuntu ALL=(sandbox) SETENV:NOPASSWD:/home/sandbox/codejail_sandbox-python3.11/bin/python +ubuntu ALL=(sandbox) SETENV:NOPASSWD:/usr/bin/find +ubuntu ALL=(ALL) NOPASSWD:/usr/bin/pkill + +Defaults!/home/sandbox/codejail_sandbox-python3.11/bin/python !requiretty +Defaults!/usr/bin/find !requiretty +Defaults!/usr/bin/pkill !requiretty diff --git a/sudoers-file/01-sandbox b/sudoers-file/01-sandbox-python-3.8 similarity index 100% rename from sudoers-file/01-sandbox rename to sudoers-file/01-sandbox-python-3.8