init arena project

guardstrikelab · Dec 7, 2023 · a100a8a · a100a8a
1 parent f31a7b0
commit a100a8a
Show file tree

Hide file tree

Showing 811 changed files with 8,514 additions and 70,304 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -0,0 +1,66 @@
+name: arenas-build
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: '${{ matrix.os }}'
+    strategy:
+      matrix:
+        os:
+          - ubuntu-20.04
+        python-version:
+          - 3.7.17
+    env:
+      TRAVIS_PULL_REQUEST: ${{ vars.TRAVIS_PULL_REQUEST }}
+      TRAVIS_BRANCH: ${{ vars.TRAVIS_BRANCH }}
+      DEPLOY_FLAG: ${{ vars.DEPLOY_FLAG }}
+      PRODUCTION_INSTANCE: ${{ secrets.PRODUCTION_INSTANCE }}
+      AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+    steps:
+      - name: 'Set up Python ${{ matrix.python-version }}'
+        uses: actions/setup-python@v2
+        with:
+          python-version: '${{ matrix.python-version }}'
+      - run: rm -f $HOME/.cache/pip/log/debug.log
+      - name: Cache multiple paths
+        uses: actions/cache@v2
+        with:
+          path: $HOME/.cache/pip
+          key: '${{ runner.os }}-${{ hashFiles(''TODO'') }}'
+      - uses: actions/checkout@v2
+      - run: sudo rm -f /etc/boto.cfg
+      - run: export CHROME_BIN=chromium-browser
+      - run: export DISPLAY=:99.0
+      - run: pip install awscli==1.18.66 importlib-metadata==4.8.3 flake8==3.8.4 coveralls==3.3.1
+      - run: flake8 ./ || exit 1;
+#      - run: >-
+#          if [ $TRAVIS_PULL_REQUEST = false ]; then echo $DOCKER_PASSWORD
+#          | docker login --username $DOCKER_USERNAME --password-stdin ||
+#          exit 1; fi
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.AWS_REGISTRY }}
+      - run: docker-compose build || exit 1;
+      - run: docker-compose run nodejs bash -c "gulp dev && karma start --single-run && gulp staging" || exit 1;
+      - run: docker-compose run -e DJANGO_SETTINGS_MODULE=settings.test django pytest --cov . --cov-config .coveragerc || exit 1;
+      - run: 'bash <(curl -s https://codecov.io/bash)'
+      - run: coveralls --rcfile=.coveragerc
+      - run: eval "$(ssh-agent -s)"
+      - run: openssl enc -d -aes-256-cbc -K ${{ secrets.ENCRYPTED_KEY }} -iv ${{ secrets.ENCRYPTED_IV }} -in scripts/deployment/evalai.pem.enc -out scripts/deployment/evalai.pem || true
+      - run: ./scripts/deployment/push.sh
+      - run: ./scripts/deployment/deploy.sh ${{ vars.DEPLOY_FLAG }}
diff --git a/README.md b/README.md
@@ -47,7 +47,17 @@ The steps are:
     git clone https://github.com/Cloud-CV/EvalAI.git evalai && cd evalai
     ```
 
-3. Build and run the Docker containers. This might take a while.
+3. Set sending email information in `settings/dev.py`.
+
+    ```shell
+    DEFAULT_FROM_EMAIL = "[email protected]"
+    EMAIL_HOST = "smtp.xxx.com"
+    EMAIL_PORT = 587
+    EMAIL_HOST_PASSWORD = "xxx"
+    EMAIL_HOST_USER = "[email protected]"
+    ```
+
+4. Build and run the Docker containers. This might take a while.
 
     ```
     docker-compose up --build

diff --git a/apps/base/utils.py b/apps/base/utils.py
@@ -34,7 +34,12 @@ def paginated_queryset(
     Return a paginated result for a queryset
     """
     paginator = pagination_class
-    paginator.page_size = settings.REST_FRAMEWORK["PAGE_SIZE"]
+    page_size = request.GET.get('page_size')
+    if page_size:
+        page_size = int(page_size)
+    else:
+        page_size = settings.REST_FRAMEWORK["PAGE_SIZE"]
+    paginator.page_size = page_size
     result_page = paginator.paginate_queryset(queryset, request)
     return (paginator, result_page)
 
@@ -46,7 +51,12 @@ def team_paginated_queryset(
     Return a paginated result for a queryset
     """
     paginator = pagination_class
-    paginator.page_size = settings.REST_FRAMEWORK["TEAM_PAGE_SIZE"]
+    page_size = request.GET.get('page_size')
+    if page_size:
+        page_size = int(page_size)
+    else:
+        page_size = settings.REST_FRAMEWORK["TEAM_PAGE_SIZE"]
+    paginator.page_size = page_size
     result_page = paginator.paginate_queryset(queryset, request)
     return (paginator, result_page)
 
@@ -252,7 +262,7 @@ def send_slack_notification(webhook=settings.SLACK_WEB_HOOK_URL, message=""):
     try:
         data = {
             "attachments": [{"color": "ffaf4b", "fields": message["fields"]}],
-            "icon_url": "https://eval.ai/dist/images/evalai-logo-single.png",
+            "icon_url": "https://arena.synkrotron.ai/images/evalai-logo-single.png",
             "text": message["text"],
             "username": "EvalAI",
         }

diff --git a/apps/challenges/aws_utils.py b/apps/challenges/aws_utils.py
@@ -56,13 +56,13 @@
     ),
     "WORKER_IMAGE": os.environ.get(
         "WORKER_IMAGE",
-        "{}.dkr.ecr.us-east-1.amazonaws.com/evalai-{}-worker:latest".format(
+        "{}.dkr.ecr.cn-northwest-1.amazonaws.com.cn/arena/evalai-{}-worker:latest".format(
             aws_keys["AWS_ACCOUNT_ID"], ENV
         ),
     ),
     "CODE_UPLOAD_WORKER_IMAGE": os.environ.get(
         "CODE_UPLOAD_WORKER_IMAGE",
-        "{}.dkr.ecr.us-east-1.amazonaws.com/evalai-{}-worker:latest".format(
+        "{}.dkr.ecr.cn-northwest-1.amazonaws.com.cn/arena/evalai-{}-worker:latest".format(
             aws_keys["AWS_ACCOUNT_ID"], ENV
         ),
     ),

diff --git a/apps/web/templates/notification_email_conformation.html b/apps/web/templates/notification_email_conformation.html
@@ -43,7 +43,7 @@ <h5><strong><a ui-sref="home" class="dark-link"><img class="auth-logo" src="{% s
         </div>
         <div class="row">
             <div class="col s12 m4 offset-m4 text-highlight center w-300">
-                <a href="https://eval.ai/api/admin" class="waves-effect waves-dark btn ev-btn-dark blue-grey darken-4">Return to Admin</a>
+                <a href="https://arena.synkrotron.ai/api/admin" class="waves-effect waves-dark btn ev-btn-dark blue-grey darken-4">Return to Admin</a>
             </div>
         </div>
     </div>

diff --git a/bower.json b/bower.json
diff --git a/docker-compose-production.yml b/docker-compose-production.yml
@@ -6,14 +6,16 @@ services:
       - "11211:11211"
 
   django:
-    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-backend:${COMMIT_ID}
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com.cn/arena/evalai-production-backend:${COMMIT_ID}
     env_file:
       - docker/prod/docker_production.env
     build:
       context: ./
       dockerfile: docker/prod/django/Dockerfile
     ports:
       - "8000:8000"
+    volumes:
+      - /etc/localtime:/etc/localtime
     logging:
       driver: awslogs
       options:
@@ -22,14 +24,16 @@ services:
         awslogs-create-group: "true"
 
   celery:
-    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-celery:${COMMIT_ID}
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com.cn/arena/evalai-production-celery:${COMMIT_ID}
     env_file:
       - docker/prod/docker_production.env
     build:
       context: ./
       dockerfile: docker/prod/celery/Dockerfile
     depends_on:
       - django
+    volumes:
+      - /etc/localtime:/etc/localtime
     logging:
       driver: awslogs
       options:
@@ -38,23 +42,27 @@ services:
         awslogs-create-group: "true"
 
   worker:
-    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-worker:${COMMIT_ID}
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com.cn/arena/evalai-production-worker:${COMMIT_ID}
     build:
       context: ./
       dockerfile: docker/prod/worker/Dockerfile
     env_file:
       - docker/prod/docker_production.env
+    volumes:
+      - /etc/localtime:/etc/localtime
 
   worker_py3.8:
-    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-worker-py3.8:${COMMIT_ID}
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com.cn/arena/evalai-production-worker-py3.8:${COMMIT_ID}
     build:
       context: ./
       dockerfile: docker/prod/worker_py3.8/Dockerfile
     env_file:
       - docker/prod/docker_production.env
+    volumes:
+      - /etc/localtime:/etc/localtime
 
   nodejs:
-    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-frontend:${COMMIT_ID}
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com.cn/arena/evalai-production-frontend:${COMMIT_ID}
     build:
       context: ./
       dockerfile: docker/prod/nodejs/Dockerfile
@@ -66,6 +74,7 @@ services:
     volumes:
       - /code/node_modules
       - /code/bower_components
+      - /etc/localtime:/etc/localtime
     logging:
       driver: awslogs
       options:
@@ -74,21 +83,25 @@ services:
         awslogs-create-group: "true"
 
   remote-worker:
-    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-remote-worker:${COMMIT_ID}
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com.cn/arena/evalai-production-remote-worker:${COMMIT_ID}
     build:
       context: ./
       dockerfile: docker/prod/worker/Dockerfile
     command: ["python", "-m", "scripts.workers.remote_submission_worker"]
     env_file:
       - docker/prod/docker_remote_worker.env
+    volumes:
+      - /etc/localtime:/etc/localtime
 
   code-upload-worker:
-    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-code-upload-worker:${COMMIT_ID}
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com.cn/arena/evalai-production-code-upload-worker:${COMMIT_ID}
     env_file:
       - docker/prod/docker_code_upload_worker.env
     build:
       context: ./
       dockerfile: docker/prod/code-upload-worker/Dockerfile
+    volumes:
+      - /etc/localtime:/etc/localtime
 
   prometheus:
     hostname: prometheus
@@ -137,7 +150,7 @@ services:
       - '9100:9100'
 
   nginx-ingress:
-    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-nginx-ingress:${COMMIT_ID}
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com.cn/arena/evalai-production-nginx-ingress:${COMMIT_ID}
     build:
       context: ./
       dockerfile: docker/prod/nginx-ingress/Dockerfile

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -27,42 +27,60 @@ services:
     volumes:
       - .:/code
 
-  worker:
-    env_file:
-      - docker/dev/docker.env
-    build:
-      context: ./
-      dockerfile: docker/dev/worker/Dockerfile
-    depends_on:
-      - django
-    volumes:
-      - .:/code
+#  worker:
+#    env_file:
+#      - docker/dev/docker.env
+#    build:
+#      context: ./
+#      dockerfile: docker/dev/worker/Dockerfile
+#    depends_on:
+#      - django
+#    volumes:
+#      - .:/code
+#
+#  worker_py3_8:
+#    env_file:
+#      - docker/dev/docker.env
+#    build:
+#      context: ./
+#      dockerfile: docker/dev/worker_py3.8/Dockerfile
+#    depends_on:
+#      - django
+#    volumes:
+#      - .:/code
 
-  worker_py3_8:
-    env_file:
-      - docker/dev/docker.env
-    build:
-      context: ./
-      dockerfile: docker/dev/worker_py3.8/Dockerfile
-    depends_on:
-      - django
-    volumes:
-      - .:/code
+#  nodejs:
+#    hostname: nodejs
+#    build:
+#      context: ./
+#      dockerfile: docker/dev/nodejs/Dockerfile
+#      args:
+#        NODE_ENV: development
+#    environment:
+#      NODE_ENV: development
+#    ports:
+#      - "8888:8888"
+#      - "35729:35729"
+#    volumes:
+#      - .:/code
+#      - /code/node_modules
+#      - /code/bower_components
 
-  nodejs:
-    hostname: nodejs
+  frontend:
+    restart: unless-stopped
     build:
       context: ./
-      dockerfile: docker/dev/nodejs/Dockerfile
+      dockerfile: docker/dev/frontend/Dockerfile
+      args:
+        NODE_ENV: development
     environment:
       NODE_ENV: development
     ports:
       - "8888:8888"
-      - "35729:35729"
+    depends_on:
+      - django
     volumes:
-      - .:/code
-      - /code/node_modules
-      - /code/bower_components
+      - ./frontend:/code
 
   statsd-exporter:
     hostname: statsd

diff --git a/docker/dev/celery/Dockerfile b/docker/dev/celery/Dockerfile
@@ -8,6 +8,6 @@ WORKDIR /code
 
 ADD requirements/* /code/
 
-RUN pip install -r dev.txt
+RUN pip install -i https://pypi.tuna.tsinghua.edu.cn/simple/ -r dev.txt
 
 CMD ["./docker/wait-for-it.sh", "django:8000", "--", "sh", "/code/docker/dev/celery/container-start.sh"]