CI: Removing External Service Dependency in CI (#228)

* dockerfile

* docker compose (it works)

* use published images to speed up

* add ci test with docker

* use compose action github action

* update docker compose file

* update compose file path

* use github-action-docker-compose-test-run

* remove unused port binding in docker-compose

* add quotes to docker compose command

* test run

* test run

* write test script in tests.sh

* use docker compose

* test run

* --rm

* ./ -> .

* test

* change to arm64

* fix docker platform problem

* change test os

* fix some build bugs

* fix runner dir

* fix a test case for sample

* update cli test to test_install

* update test benchmark to improve coverage

* remove unused and maintain structured output compatibility

* fix evaluator bug

* add a test script which contributors can run locally

* update contribution guides

* add custom model doc (#215)

.devcontainer/Dockerfile

@@ -0,0 +1,6 @@
+FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye
+
+RUN pip install uv\
+   && uv venv /workspaces/.venv\
+   && export UV_PROJECT_ENVIRONMENT=/workspaces/.venv\
+   && echo export UV_PROJECT_ENVIRONMENT=/workspaces/.venv >> /root/.bashrc

.devcontainer/Dockerfile.llamacpp

@@ -0,0 +1,9 @@
+FROM ghcr.io/ggerganov/llama.cpp:server
+
+# Install curl and other necessary utilities
+RUN apt-get update && \
+    apt-get install -y curl && \
+    rm -rf /var/lib/apt/lists/* && \
+    curl -L https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q8_0.gguf --output Llama-3.2.gguf
+
+CMD ["-m", "Llama-3.2.gguf", "--port", "8000"]

.devcontainer/devcontainer.json

@@ -3,11 +3,10 @@
 {
 	"name": "Sotopia Place",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye",
-	"features": {
-		"ghcr.io/itsmechlark/features/redis-server:1": {},
-		"ghcr.io/prulloac/devcontainer-features/ollama:1": {}
-	},
+
+	"dockerComposeFile": "docker-compose.yml",
+	"service": "devcontainer",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	// "features": {},
@@ -16,8 +15,7 @@
 	// "forwardPorts": [],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand":
-       "pip install uv; uv venv /workspaces/.venv; export UV_PROJECT_ENVIRONMENT=/workspaces/.venv; echo export UV_PROJECT_ENVIRONMENT=/workspaces/.venv >> /root/.bashrc; uv sync --all-extras; ollama pull llama3.2:1b",
+	"postCreateCommand": "uv sync --all-extras",
 
 	// Configure tool-specific properties.
 	// "customizations": {},

.devcontainer/docker-compose.yml

@@ -0,0 +1,31 @@
+services:
+  devcontainer:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      llamacpp:
+        condition: service_started
+      redis:
+        condition: service_healthy
+    volumes:
+      - ../..:/workspaces:cached
+    command: sleep infinity
+    network_mode: "host"
+
+  llamacpp:
+    build:
+      context: .
+      dockerfile: Dockerfile.llamacpp
+    container_name: llamacpp
+    network_mode: "host"
+
+  redis:
+    image: redis/redis-stack-server:latest
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+    network_mode: "host"
+
+networks:
+  sotopia:
+    external: false

.github/workflows/tests.yml → .github/workflows/cli_tests.yml

@@ -1,4 +1,4 @@
-name: Pytest
+name: Pytest (Installation)
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -40,12 +40,8 @@ jobs:
           python -m pip install uv
           uv sync --extra test --extra chat
     - name: Test with pytest
-      env: # Or as an environment variable
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        REDIS_OM_URL: ${{ secrets.REDIS_OM_URL }}
-        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
       run: |
-        uv run pytest --cov=. --cov-report=xml
+        uv run pytest tests/cli/test_install.py --cov=. --cov-report=xml
     - name: Upload coverage report to Codecov
       uses: codecov/[email protected]
       with:

.github/workflows/tests.sh

@@ -0,0 +1 @@
+uv run --extra test --extra chat pytest --ignore tests/cli --cov=. --cov-report=xml

.github/workflows/tests_in_docker.yml

@@ -0,0 +1,35 @@
+name: Pytest in docker
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+on:
+  push:
+    branches:
+      - main
+      - release
+      - dev
+  pull_request:
+    branches:
+      - main
+      - release
+
+jobs:
+  Pytest:
+    strategy:
+      max-parallel: 5
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Docker Compose
+      run: docker compose -f .devcontainer/docker-compose.yml up -d
+    - name: Run tests
+      run: docker compose -f .devcontainer/docker-compose.yml run --rm -u root -v /home/runner/work/sotopia/sotopia:/workspaces/sotopia devcontainer /bin/sh -c "cd /workspaces/sotopia; ls; uv sync --extra test --extra chat; uv run pytest --ignore tests/cli --cov=. --cov-report=xml"
+    - name: Upload coverage report to Codecov
+      uses: codecov/[email protected]
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

docs/pages/concepts/generation.mdx

@@ -42,3 +42,9 @@ The input to `agenerate` are the following:
 - `output_parser`: The output parser to use. The output parser should be a subclass of `BaseOutputParser`.
 
 In this example, we generate a list of the first `n` prime numbers with the `gpt-4o` model. The `ListOfIntOutputParser` is used to parse the output into a list of integers.
+
+### Using custom models
+
+Apart from using api endpoints from LLM providers like OpenAI, Together AI, Azure, etc., you can also use custom model with OpenAI compatible endpoints.
+You will need to set the model name to `custom/<model_name>@url`, and CUSTOM_API_KEY to the API key of the custom model.
+For an example, check out `examples/generation_api/custom_model.py`.

docs/pages/contribution/contribution.md

@@ -10,88 +10,70 @@ There are many ways that you can contribute:
 2. **Send feedback** after each session by using our feedback mechanisms (if implemented), so we can see where things are working and failing, and also build an open dataset for training social agents.
 3. **Improve the Codebase** by sending PRs (see details below). In particular, we have some [good first issue](https://github.com/sotopia-lab/sotopia/labels/good%20first%20issue) issues that may be ones to start on.
 
-## Understanding Sotopia's CodeBase
+## For Sotopia Developers
+If you are a developer and want to contribute to Sotopia, we really really appreciate it. Here are some guidelines for you. The same guidelines also apply to Sotopia team members.
 
-To understand the codebase, please refer to the README in each module:
-- [sotopia](./sotopia/README.md)
-- [docs](./docs/README.md)
-- [tests](./tests/README.md)
+### 0. Before You Start
 
-When you write code, it is also good to write tests. Please navigate to the `tests` folder to see existing test suites.
-At the moment, we have different kinds of tests including unit tests and integration tests. Please refer to the README for each test suite. These tests also run on GitHub's continuous integration to ensure quality of the project.
+Before you start contributing to Sotopia, please make sure you have access to the following:
 
-## Sending Pull Requests to Sotopia
+- Python environment supporting Python 3.10+
+- Uv: You can install uv using `pip install uv;` Sotoipa uses uv for managing the project.
+- Docker: Sotopia uses Docker for testing and deployment. You will need Docker to run the tests.
+- Redis: Sotopia uses Redis for caching and storing data. You will need to have a Redis server running locally or remotely.
+- LLM server or API key: Most of Sotopia applications require access to a language model. You can use OpenAI, Anthropic, or other language models. If you don't have access to a language model, you can use a local model like Ollama, Llama.cpp, or vLLM.
 
-### 1. Fork the Official Repository
-Fork the [Sotopia repository](https://github.com/sotopia-lab/sotopia) into your own account.
-Clone your own forked repository into your local environment:
+Don't want to set up all these environments manually? Follow the steps in [here](#set-up-the-development-environment) to set up a Dev Container.
 
-```shell
-git clone [email protected]:<YOUR-USERNAME>/sotopia.git
-```
-
-### 2. Configure Git
+### 1. Code Quality
 
-Set the official repository as your [upstream](https://www.atlassian.com/git/tutorials/git-forks-and-upstreams) to synchronize with the latest update in the official repository.
-Add the original repository as upstream:
+Sotopia code is typed, linted, and (mostly) tested. When you send a pull request, please make sure to run the following commands to check your code:
 
+#### Mypy
+Sotopia uses [mypy](https://mypy.readthedocs.io/en/stable/) for static type checking. You can run mypy by:
 ```shell
-cd sotopia
-git remote add upstream [email protected]:sotopia-lab/sotopia.git
+uv run mypy --strict .
 ```
 
-Verify that the remote is set:
+For most IDEs, you can also install the mypy extension to check the type while you are coding.
+
+#### Linting
+Sotopia uses several pre-commit hooks to ensure code quality. You can install pre-commit by:
 
 ```shell
-git remote -v
+uv run pre-commit install
 ```
 
-You should see both `origin` and `upstream` in the output.
+Then, every time you commit, pre-commit will run the hooks to check your code. Sotopia has a github action
+which autofixes some of the issues, but it is better to fix them before committing.
 
-### 3. Synchronize with Official Repository
-Synchronize latest commit with official repository before coding:
+#### Unit and Integration Tests
+Sotopia provides a docker environment for testing. You can run the following command to test your code:
 
 ```shell
-git fetch upstream
-git checkout main
-git merge upstream/main
-git push origin main
+# Starting from your local sotopia repository
+# Requires Docker
+# If you are using devcontainer, you can run pytest --ignore tests/cli
+bash tests/tests.sh
 ```
 
-### 4. Set up the Development Environment
-
-We recommend using Dev Containers to set up your development environment, but note that Docker is required for install Dev Containers.
-
-#### Using VSCode
-
-If you use VSCode, you can install the Dev Containers extension, and then in [Command Palette](https://code.visualstudio.com/docs/getstarted/userinterface#_command-palette), run `Dev Containers: Open Folder in Container`.
-After the container is built (the first time it may take 10+ minutes to build), you have a Redis server and local Llama server running.
-
-#### Other IDEs or Editors
-
-Please refer to [Dev Containers](https://containers.dev/supporting#editors) to see how to set up DevContainers in other editors.
-
-#### Without Dev Containers
-
-You can also set up the development environment without Dev Containers. There are three things you will need to set up manually:
-
-- Python and uv: Please start from an environment supporting Python 3.10+ and install uv using `pip install uv; uv sync --all-extra`.
-- Redis: Please refer to introduction page for the set up of Redis.
-- Local LLM (optional): If you don't have access to model endpoints (e.g. OpenAI, Anthropic or others), you can use a local model. You can use Ollama, Llama.cpp,  vLLM or many others which support OpenAI compatible endpoints.
+The above command will run all tests in the `tests` folder except for `tests/cli`. If you are not changing the installation CLI, you don't need to worry about it.
 
+If you have implemented a new feature, please also write tests for it. Sotopia github action will
+check the code coverage after your PR passed the tests. The PR will not be merged if the code coverage is lower than the threshold.
+Feel free to ask on Discord if you find it hard to write tests for your features.
 
-### 5. Write Code and Commit It
+#### Documentation
+Lastly, please also update the documentation if you have changed the API or added new features. You can find the documentation in the `docs` folder.
 
-Once you have done this, you can write code, test it, and commit it to a branch (replace `my_branch` with an appropriate name):
+To build the documentation, you can run:
 
 ```shell
-git checkout -b my_branch
-git add .
-git commit
-git push origin my_branch
+# You will need to install [bun](https://bun.sh/) first
+cd docs; bun run dev
 ```
 
-### 6. Open a Pull Request
+### How to Open a Pull Request
 
 * On GitHub, go to the page of your forked repository, and create a Pull Request:
    - Click on `Branches`
@@ -104,9 +86,8 @@ The PR should appear in [Sotopia PRs](https://github.com/sotopia-lab/sotopia/pul
 
 Then the Sotopia team will review your code.
 
-## PR Rules
+#### Some PR Rules
 
-### 1. Pull Request title
 As described [here](https://github.com/commitizen/conventional-commit-types/blob/master/index.json), a valid PR title should begin with one of the following prefixes:
 
 - `feat`: A new feature
@@ -127,6 +108,42 @@ For example, a PR title could be:
 
 You may also check out previous PRs in the [PR list](https://github.com/sotopia-lab/sotopia/pulls).
 
-### 2. Pull Request description
+For the description of the PR:
 - If your PR is small (such as a typo fix), you can go brief.
 - If it contains a lot of changes, it's better to write more details.
+- If your PR is related to an issue, please mention the issue number in the description.
+
+## Contribution Tips
+The following are some tips for contributing to Sotopia, you are not required to follow them, but they may help you to contribute more effectively.
+
+### Set up the Development Environment
+
+We recommend using Dev Containers to set up your development environment, but note that Docker is required for install Dev Containers.
+
+#### Using VSCode
+
+If you use VSCode, you can install the Dev Containers extension, and then in [Command Palette](https://code.visualstudio.com/docs/getstarted/userinterface#_command-palette), run `Dev Containers: Open Folder in Container`.
+After the container is built (the first time it may take 10+ minutes to build), you have a Redis server and local Llama server running.
+
+#### Other IDEs or Editors
+
+Please refer to [Dev Containers](https://containers.dev/supporting#editors) to see how to set up DevContainers in other editors.
+
+#### Without Dev Containers
+
+You can also set up the development environment without Dev Containers. There are three things you will need to set up manually:
+
+- Python and uv: Please start from an environment supporting Python 3.10+ and install uv using `pip install uv; uv sync --all-extra`.
+- Redis: Please refer to introduction page for the set up of Redis.
+- Local LLM (optional): If you don't have access to model endpoints (e.g. OpenAI, Anthropic or others), you can use a local model. You can use Ollama, Llama.cpp,  vLLM or many others which support OpenAI compatible endpoints.
+
+
+### Modern Editors
+You might love vim/emacs/sublime/Notebook++. But modern editors like VSCode, PyCharm offers more features that can help you
+write better code faster. Here are some extensions that you might want to install:
+
+- Mypy Type Checker: For static type checking. Make sure to choose the right python interpreter (`/workspaces/.venv/bin/python` in the devcontainer or `.venv` locally) in the settings, and enable `Mypy: Run Using Active Interpreter`.
+- Dev Containers: If you are using Dev Containers, you can install the Dev Containers as mentioned above.
+- Ruff: For formatting your code.
+- MDX: For the documentation website under `docs` folder.
+- Even Better TOML: For editing aact workflow files.

examples/generation_api/custom_model.py

@@ -3,13 +3,12 @@
 # 1. Use the sotopia devcontainer and run the following command in the terminal:
 #        uv run python examples/generation_api/custom_model.py
 #    This example can also serve a sanity check for your devcontainer setup.
-# OR 2. after installing sotopia, install ollama, and then:
-#        ollama pull llama3.2:1b; python examples/generation_api/custom_model.py
+# OR 2. after installing sotopia, install llama.cpp, and then follow llama.cpp's instructions to serve the model
 # OR 3. after installing sotopia, serve your desired model on your desired port,
 #    and then change the model_name of `agenerate` to point to your desired model.
 #    Finally:
 #        python examples/generation_api/custom_model.py
-# Expected output for (1 and 2): a bunch of logs and an output [[14, 67], [6, 8, 3], [6, 8, 3, 9], [6, 8, 3, 9, 7], [7, 9, 6, 8, 4, 1]]
+# Expected output for 1: a bunch of logs and an output [[14, 7], [14, 7, 3], [14, 7, 3, 9], [14, 7, 3, 9, 6], [14, 7, 3, 9, 6, 8]]
 
 from sotopia.generation_utils.generate import ListOfIntOutputParser, agenerate
 import logging
@@ -20,7 +19,7 @@
 
 async def generate_n_random_numbers(n: int) -> list[int]:
     return await agenerate(
-        model_name="custom/llama3.2:1b@http://localhost:11434/v1",
+        model_name="custom/llama3.2:1b@http://localhost:8000/v1",
         template="Generate {n} random integer numbers. {format_instructions}",
         input_values={"n": str(n)},
         temperature=0.0,

sotopia/database/__init__.py

@@ -1,5 +1,5 @@
 from typing import TypeVar
-from redis_om import JsonModel
+from redis_om import JsonModel, Migrator
 from .annotators import Annotator
 from .env_agent_combo_storage import EnvAgentComboStorage
 from .logs import AnnotationForEpisode, EpisodeLog
@@ -31,6 +31,10 @@
 from .waiting_room import MatchingInWaitingRoom
 from .aggregate_annotations import map_human_annotations_to_episode_logs
 
+from logging import Logger
+
+logger = Logger("sotopia.database")
+
 __all__ = [
     "AgentProfile",
     "EnvironmentProfile",
@@ -71,3 +75,10 @@ def _json_model_all(cls: type[InheritedJsonModel]) -> list[InheritedJsonModel]:
 
 
 JsonModel.all = classmethod(_json_model_all)  # type: ignore[assignment,method-assign]
+
+try:
+    Migrator().run()
+except Exception as e:
+    logger.debug(
+        f"Error running migrations: {e} This is expected if you have not set up redis yet."
+    )

sotopia/envs/evaluators.py

@@ -315,6 +315,7 @@ async def __acall__(
                     pydantic_object=self.response_format_class
                 ),
                 temperature=temperature,
+                structured_output=self.model_name.startswith("custom/structured"),
             )
             response_list = []
             # TODO: multiple agents
@@ -345,6 +346,7 @@ async def __acall__(
                 )
             return response_list
         except Exception as e:
+            print(e)
             log.debug(f"[red] Failed to generate environment response. {e}")
             return []