Merge branch 'multi_dimensional_action' into boptest-gym-service

.github/workflows/github-actions.yml

@@ -1,5 +1,10 @@
 name: CI of BOPTEST-Gym using GitHub Actions
-on: [push]
+on:
+  push:
+  pull_request:
+    branches:
+      - main
+    types: [opened, synchronize, reopened]
 jobs:
   test-local:
     runs-on: ubuntu-latest
@@ -11,12 +16,32 @@ jobs:
         uses: actions/checkout@v3
       - name: Pull boptestgym image from registry
         run: make pull-boptestgym
-      - run: echo "The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
-      - name: List of files in the repository
-        run: | 
-         ls ${{ github.workspace }}
+      - name: Pull boptest_base image from registry
+        run: make pull-boptestbase
+      - name: Install Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
       - name: Test local version
         run: make test-local-in-container
+  test-multiaction:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: testing
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Pull boptestgym image from registry
+        run: make pull-boptestgym
+      - name: Pull boptest_base image from registry
+        run: make pull-boptestbase
+      - name: Install Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
+      - name: Test multi-action
+        run: make test-multiaction-in-container
   test-vectorized:
     runs-on: ubuntu-latest
     defaults:
@@ -27,10 +52,12 @@ jobs:
         uses: actions/checkout@v3
       - name: Pull boptestgym image from registry
         run: make pull-boptestgym
-      - run: echo "The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
-      - name: List of files in the repository
-        run: | 
-         ls ${{ github.workspace }}
+      - name: Pull boptest_base image from registry
+        run: make pull-boptestbase
+      - name: Install Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
       - name: Test vectorized environment
         run: make test-vectorized-in-container
   test-service:
@@ -43,10 +70,8 @@ jobs:
         uses: actions/checkout@v3
       - name: Pull boptestgym image from registry
         run: make pull-boptestgym
-      - run: echo "The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
-      - name: List of files in the repository
-        run: | 
-         ls ${{ github.workspace }}
+      - name: Pull boptest_base image from registry
+        run: make pull-boptestbase
       - name: Test service version
         run: make test-service-in-container
 

boptestGymEnv.py

@@ -1040,6 +1040,41 @@ def __init__(self, env, n_bins_act=10):
 
         # Instantiate discretized action space
         self.action_space = spaces.Discrete((n_bins_act+1) ** self.n_act)
+
+    def _get_indices(self, action_wrapper):
+        """
+        Returns the indices of the discretized action space corresponding to the given action wrapper.
+        
+        Parameters
+        ----------
+        action_wrapper : int
+            The action wrapper value to be converted to indices.
+        
+        Returns
+        -------
+        list
+            A list of indices representing the discretized action space.
+
+        Example
+        -------
+        Suppose:
+        self.n_act = 3 (number of actions)
+        self.n_bins_act = 4 (number of bins per action)
+        self.val_bins_act = [[0, 1, 2, 3], [10, 11, 12, 13], [20, 21, 22, 23]] (value bins for each action)
+        
+        Then, `_get_indices` example, for action_wrapper = 37:
+        indices = []
+        Loop 3 times:
+        Iteration 1: indices.append(37 % 4) -> indices = [1], action_wrapper //= 4 -> action_wrapper = 9
+        Iteration 2: indices.append(9 % 4) -> indices = [1, 1], action_wrapper //= 4 -> action_wrapper = 2
+        Iteration 3: indices.append(2 % 4) -> indices = [1, 1, 2], action_wrapper //= 4 -> action_wrapper = 0
+        Reverse indices: [2, 1, 1]
+        """
+        indices=[]
+        for _ in range(self.n_act):
+            indices.append(action_wrapper%self.n_bins_act)
+            action_wrapper //= self.n_bins_act
+        return indices[::-1]    
 
     def action(self, action_wrapper):
         '''This method accepts a single parameter (the modified action
@@ -1058,17 +1093,28 @@ def action(self, action_wrapper):
         
         Notes
         -----
-        To better understand what this method needs to do, see how the 
+        To better understand what this method needs to do, see what the 
         `gym.ActionWrapper` parent class is doing in `gym.core`:
         
         Implement something here that performs the following mapping:
         DiscretizedObservationWrapper.action_space --> DiscretizedActionWrapper.action_space
-        
+
+        Example
+        -------
+        For action_wrapper = 37 (follows the example of `_get_indices` above):
+
+        indices = [2, 1, 1]
+        Map indices to action values:
+        bins[2] from [0, 1, 2, 3] -> 2
+        bins[1] from [10, 11, 12, 13] -> 11
+        bins[1] from [20, 21, 22, 23] -> 21
+        Convert to NumPy array: np.asarray([2, 11, 21])
+        Return action: [2, 11, 21]
         '''
-
+        indices = self._get_indices(action_wrapper)
         # Get the action values from bin indexes
         action = [bins[x]
-                  for x, bins in zip(action_wrapper.flatten(), 
+                  for x, bins in zip(indices, 
                                      self.val_bins_act)]
 
         action = np.asarray(action).astype(self.env.action_space.dtype)

releasenotes.md

@@ -1,6 +1,13 @@
 # Release Notes
 
-BOPTEST-Gym has two main dependencies: BOPTEST and Stable-Baselines3. For simplicity, the first two digits of the version number match the same two digits of the BOPTEST version of which BOPTEST-Gym is compatible with. For example, BOPTEST-Gym v0.3.x is compatible with BOPTEST v0.3.x. The last digit is reserved for other internal edits specific to this repository only. See [here](https://github.com/ibpsa/project1-boptest/blob/master/releasenotes.md) for BOPTEST release notes. 
+BOPTEST-Gym has two main dependencies: BOPTEST and Stable-Baselines3. For simplicity, the first two digits of the version number match the same two digits of the BOPTEST version of which BOPTEST-Gym is compatible with. For example, BOPTEST-Gym v0.6.x is compatible with BOPTEST v0.6.x. The last digit is reserved for other internal edits specific to this repository only. See [here](https://github.com/ibpsa/project1-boptest/blob/master/releasenotes.md) for BOPTEST release notes. 
+
+
+## BOPTEST-Gym v0.6.0-dev
+
+Released on xx/xx/xxxx.
+
+- Support for multi-dimensional action spaces. A multi-dimensional action space is tested in the `singlezone_commercial_hydronic` test case. This is for [#19](https://github.com/ibpsa/project1-boptest-gym/issues/19). 
 
 ## BOPTEST-Gym v0.6.0
 

testing/Makefile

@@ -2,91 +2,116 @@
 ROOT ?= $(shell dirname \
   $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))))
 
-# Local image name and home
-IMG_NAME=boptestgym
-IMG_HOME=/home/developer/boptestgym
+# Local image name, home directory, and remote registry for BOPTEST-Gym
+IMG_NAME_BOPTESTGYM=boptestgym
+IMG_HOME_BOPTESTGYM=/home/developer/boptestgym
+IMG_REGI_BOPTESTGYM=javierarroyo/boptestgym
 
-# Name of remote registry image
-IMG_REGI=javierarroyo/boptestgym
+# Local image name and remote registry for the BOPTEST test case
+IMG_NAME_BOPTESTBASE=boptest_base
+IMG_REGI_BOPTESTBASE=javierarroyo/boptest_base
 
 # BOPTEST commit used for the tests
 BOPTEST_COMMIT=78665506a620fc07bc2a8f301661aef3dd6c88c2
 
 # Define current BOPTEST-Gym version (should be even with BOPTEST version defined in commit above)
-VERSION = 0.6.0
+VERSION = 0.6.0-dev
 
 build-boptestgym:
 	docker build -f ${ROOT}/testing/Dockerfile \
-		--progress=plain --rm -t ${IMG_NAME} .
+		--progress=plain --rm -t ${IMG_NAME_BOPTESTGYM} .
 
 build-boptestgym-no-cache:
 	docker build -f ${ROOT}/testing/Dockerfile \
-		--progress=plain --no-cache --rm -t ${IMG_NAME} .
-
+		--progress=plain --no-cache --rm -t ${IMG_NAME_BOPTESTGYM} .
+
+# Build the generic BOPTEST base image without mounting any test case
+build-boptestbase:
+	make download-boptest
+	cd project1-boptest-${BOPTEST_COMMIT} && \
+	docker compose build
+
 run-boptestgym:
 	docker run \
-		--name ${IMG_NAME} \
+		--name ${IMG_NAME_BOPTESTGYM} \
 		--detach=false \
 		--network=host \
 		--rm \
 		--user $(id -u):$(id -g) \
-		-v ${ROOT}:${IMG_HOME}:rw \
-		-w ${IMG_HOME}/testing \
+		-v ${ROOT}:${IMG_HOME_BOPTESTGYM}:rw \
+		-w ${IMG_HOME_BOPTESTGYM}/testing \
 		-it \
-		${IMG_NAME}
+		${IMG_NAME_BOPTESTGYM}
 
 run-boptestgym-detached:
 	docker run \
-		--name ${IMG_NAME} \
+		--name ${IMG_NAME_BOPTESTGYM} \
 		--detach=true \
 		--network=host \
 		--rm \
 		--user $(id -u):$(id -g) \
-		-v ${ROOT}:${IMG_HOME}:rw \
-		-w ${IMG_HOME}/testing \
+		-v ${ROOT}:${IMG_HOME_BOPTESTGYM}:rw \
+		-w ${IMG_HOME_BOPTESTGYM}/testing \
 		-it \
-		${IMG_NAME}
+		${IMG_NAME_BOPTESTGYM}
 
 stop-boptestgym:
-	docker stop ${IMG_NAME}
+	docker stop ${IMG_NAME_BOPTESTGYM}
 
 exec-boptestgym:
 	docker exec \
 		-i \
-		${IMG_NAME} \
+		${IMG_NAME_BOPTESTGYM} \
 		/bin/bash -c "${ARGS} && exit"
 
 push-boptestgym:
 # requires `docker login` first
-	docker tag ${IMG_NAME} ${IMG_REGI}:${VERSION}
-	docker push ${IMG_REGI}:${VERSION}
+	docker tag ${IMG_NAME_BOPTESTGYM} ${IMG_REGI_BOPTESTGYM}:${VERSION}
+	docker push ${IMG_REGI_BOPTESTGYM}:${VERSION}
 
 pull-boptestgym:
-	docker pull ${IMG_REGI}:${VERSION}
-	docker tag ${IMG_REGI}:${VERSION} ${IMG_NAME}
+	docker pull ${IMG_REGI_BOPTESTGYM}:${VERSION}
+	docker tag ${IMG_REGI_BOPTESTGYM}:${VERSION} ${IMG_NAME_BOPTESTGYM}
 
-make download-boptest:
+push-boptestbase:
+# requires `docker login` first
+	docker tag ${IMG_NAME_BOPTESTBASE} ${IMG_REGI_BOPTESTBASE}:${VERSION}
+	docker push ${IMG_REGI_BOPTESTBASE}:${VERSION}
+
+pull-boptestbase:
+	docker pull ${IMG_REGI_BOPTESTBASE}:${VERSION}
+	docker tag ${IMG_REGI_BOPTESTBASE}:${VERSION} ${IMG_NAME_BOPTESTBASE}
+
+download-boptest:
 	curl -L -o boptest.zip https://github.com/ibpsa/project1-boptest/archive/${BOPTEST_COMMIT}.zip
 	unzip -o -q boptest.zip
 
 run-boptest-case:
 	make download-boptest
 	cd project1-boptest-${BOPTEST_COMMIT} && \
-	TESTCASE=bestest_hydronic_heat_pump docker-compose up -d --quiet-pull
+	TESTCASE=bestest_hydronic_heat_pump docker compose up -d
+	sleep 10
 
 run-boptest-case-no-cache:
 	make download-boptest
 	cd project1-boptest-${BOPTEST_COMMIT} && \
-	TESTCASE=bestest_hydronic_heat_pump docker-compose up -d --force-recreate --build
+	TESTCASE=bestest_hydronic_heat_pump docker compose up -d --force-recreate --build
+	sleep 10
+
+run-boptest-case-commercial:
+	make download-boptest
+	cd project1-boptest-${BOPTEST_COMMIT} && \
+	TESTCASE=singlezone_commercial_hydronic docker compose up -d && \
+	sleep 10
 
 run-boptest-vectorized:
 	make download-boptest && \
 	cd .. && python3 generateDockerComposeYml.py testing/project1-boptest-${BOPTEST_COMMIT} && \
 	cd testing/project1-boptest-${BOPTEST_COMMIT} && \
-	TESTCASE=bestest_hydronic_heat_pump docker-compose up -d --quiet-pull
+	TESTCASE=bestest_hydronic_heat_pump docker compose up -d 
 
 stop-boptest-case:
-	cd project1-boptest-${BOPTEST_COMMIT} && docker-compose down
+	cd project1-boptest-${BOPTEST_COMMIT} && docker compose down
 
 cleanup-boptest:
 	rm boptest.zip
@@ -97,6 +122,9 @@ cleanup-boptest:
 test-local:
 	python3 -m unittest test_boptestGymEnv.BoptestGymEnvTest
 
+test-multiaction:
+	python3 -m unittest test_boptestGymEnv.BoptestGymEnvMultiActTest
+
 # Vectorized needs to run separate since modifies docker-compose.yml to have multiple boptest instances 
 test-vectorized:
 	python3 -m unittest test_boptestGymEnv.BoptestGymVecTest project1-boptest-${BOPTEST_COMMIT}
@@ -113,6 +141,14 @@ test-local-in-container:
 	make stop-boptest-case
 	make cleanup-boptest
 
+test-multiaction-in-container:
+	make run-boptest-case-commercial
+	make run-boptestgym-detached
+	make exec-boptestgym ARGS="make test-multiaction"
+	make stop-boptestgym
+	make stop-boptest-case
+	make cleanup-boptest
+
 test-vectorized-in-container:
 	make run-boptest-vectorized
 	make run-boptestgym-detached

testing/references/multiaction_training.csv

@@ -0,0 +1,2 @@
+keys,value
+0,439

testing/references/vectorized_training.csv

@@ -1,2 +1,2 @@
 keys,value
-0,0
+0,7

testing/test_boptestGymEnv.py

@@ -13,7 +13,7 @@
 import shutil
 from testing import utilities
 from examples import run_baseline, run_sample, run_save_callback,\
-    run_variable_episode, run_vectorized, train_RL
+    run_variable_episode, run_vectorized, run_multiaction, train_RL
 from collections import OrderedDict
 from boptestGymEnv import BoptestGymEnv
 from stable_baselines3.common.env_checker import check_env
@@ -619,5 +619,30 @@ def check_from_cell_output(self, cell_output, str_output):
         # Check results
         self.compare_ref_json(out_json, file_ref)
 
+class BoptestGymEnvMultiActTest(unittest.TestCase, utilities.partialChecks):
+    ''' Test multi-action training with the `singlezone_commercial_hydronic`
+    test case. 
+    '''
+
+
+    def test_training_multi_action(self):
+        '''Checks an estimated action after an agent is trained in a multi-action environment.'''
+
+        # Train an agent in a multi-action environment.
+        self.env, model = run_multiaction.train_multiaction()
+
+        # Test one step with the trained model
+        obs = self.env.reset()[0]
+        df = pd.DataFrame([model.predict(obs)[0]], columns=['value'])
+        df.index.name = 'keys'
+        ref_filepath    = os.path.join(utilities.get_root_path(), 
+                            'testing', 'references', 'multiaction_training.csv')
+        self.compare_ref_values_df(df, ref_filepath)
+
+    def tearDown(self):
+        '''Clean up after each test.'''
+        self.env.close()
+
+
 if __name__ == '__main__':
     utilities.run_tests(os.path.basename(__file__))