diff --git a/.gitignore b/.gitignore index 9ab98dd..12e684b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ logs out-comptests sq.db challenges-dir +evaluator.log +output-local-evaluation diff --git a/LF.challenge.yaml b/LF.challenge.yaml index 4a8fb9a..5b3b64a 100644 --- a/LF.challenge.yaml +++ b/LF.challenge.yaml @@ -1,88 +1,31 @@ -challenge: aido2_LF_r3-v4-z2 +challenge: aido2-LF-sim-validation -title: "[AIDO 2] LF - Lane following." -tags: [aido2, aido2-embodied, LF, v4] +title: "LF 🚗 - Lane following (simulation 👾, validation 🏋)" +tags: [visible, ml-validation, aido2, aido2-embodied, LF, v4] description: | Lane following challenge. protocol: aido2_db18_agent-z2 -date-open: 2018-12-07 -date-close: 2019-12-07 - -roles: {} - -scoring: - scores: - - - name: driven_lanedir_consec_median - short: Traveled distance - description: | - This is the median distance traveled, *along a lane*. - (That is, going in circles will not make this metric increase.) - - This is discretized to tiles. - order: higher-is-better - discretization: 0.62 - - - name: survival_time_median - short: Survival time - description: | - This is the median survival time. The simulation - is terminated when the car goes outside of the road - or it crashes with an obstacle. - order: higher-is-better - discretization: 2.0 - - - name: deviation-center-line_median - short: Lateral deviation - description: | - This is the median lateral deviation from the center line. - order: lower-is-better - discretization: 0.01 - - - name: in-drivable-lane_median - short: Major infractions - description: | - This is the median of the time spent outside of the drivable zones. - For example this penalizes driving in the wrong lane. - order: lower-is-better - discretization: 0.2 - +date-open: 2019-04-10 +date-close: 2019-05-15 +scoring: !include scoring.yaml steps: step1-simulation: title: Simulation step description: "" - timeout: 10 + timeout: 1200 # 20 minutes evaluation_parameters: services: - evaluator: - environment: - DISABLE_CONTRACTS: 1 - AIDONODE_ENCODING: cbor - experiment_manager_parameters: | - episodes_per_scenario: 1 - episode_length_s: 15.0 - min_episode_length_s: 0.0 - seed: 42 - physics_dt: 0.05 - max_failures: 2 - agent_in: /fifos/agent-in - agent_out: /fifos/agent-out - sim_in: /fifos/simulator-in - sim_out: /fifos/simulator-out - sm_in: /fifos/scenario_maker-in - sm_out: /fifos/scenario_maker-out - build: - context: ./experiment_manager + evaluator: !include comp_evaluator.yaml + simulator: !include comp_simulator.yaml + solution: !include comp_solution.yaml scenario_maker: environment: - DISABLE_CONTRACTS: 1 - AIDONODE_ENCODING: cbor AIDONODE_DATA_IN: /fifos/scenario_maker-in AIDONODE_DATA_OUT: fifo:/fifos/scenario_maker-out AIDONODE_CONFIG: | @@ -93,34 +36,3 @@ steps: build: context: ./scenario_maker - simulator: - build: - context: ./simulator - environment: - AIDONODE_DATA_IN: /fifos/simulator-in - AIDONODE_DATA_OUT: fifo:/fifos/simulator-out - AIDONODE_ENCODING: cbor - AIDONODE_CONFIG: | - env_constructor: Simulator - env_parameters: - max_steps: 500001 # we don't want the gym to reset itself - domain_rand: 0 - camera_width: 640 - camera_height: 480 - distortion: true - - solution: - image: SUBMISSION_CONTAINER - environment: - AIDONODE_DATA_IN: /fifos/agent-in - AIDONODE_DATA_OUT: fifo:/fifos/agent-out - AIDONODE_ENCODING: cbor - - - - -transitions: - - [START, success, step1-simulation] - - [step1-simulation, success, SUCCESS] - - [step1-simulation, failed, FAILED] - - [step1-simulation, error, ERROR] diff --git a/LFV.challenge.yaml b/LFV.challenge.yaml index e139a76..dbda80e 100644 --- a/LFV.challenge.yaml +++ b/LFV.challenge.yaml @@ -1,53 +1,16 @@ -challenge: aido2_LFV_r3-v4-z2 +challenge: aido2-LFV-sim-validation -title: "[AIDO 2] LFV - Lane following with Vehicles " -tags: [aido2, aido2-embodied, LF, v4] +title: "LFV 🚗🚗 - Lane following + Vehicles (simulation 👾, validation 🏋)" +tags: [visible, ml-validation, aido2, aido2-embodied, LFV, v4] description: | Lane following challenge. protocol: aido2_db18_agent-z2 -date-open: 2018-12-07 -date-close: 2019-12-07 +date-open: 2019-04-10 +date-close: 2019-05-15 -roles: {} - -scoring: - scores: - - - name: driven_lanedir_consec_median - short: Traveled distance - description: | - This is the median distance traveled, *along a lane*. - (That is, going in circles will not make this metric increase.) - - This is discretized to tiles. - order: higher-is-better - discretization: 0.62 - - - name: survival_time_median - short: Survival time - description: | - This is the median survival time. The simulation - is terminated when the car goes outside of the road - or it crashes with an obstacle. - order: higher-is-better - discretization: 2.0 - - - name: deviation-center-line_median - short: Lateral deviation - description: | - This is the median lateral deviation from the center line. - order: lower-is-better - discretization: 0.01 - - - name: in-drivable-lane_median - short: Major infractions - description: | - This is the median of the time spent outside of the drivable zones. - For example this penalizes driving in the wrong lane. - order: lower-is-better - discretization: 0.2 +scoring: !include scoring.yaml @@ -56,28 +19,13 @@ steps: title: Simulation step description: "" - timeout: 10 + timeout: 1200 # 20 minutes evaluation_parameters: services: - evaluator: - environment: - DISABLE_CONTRACTS: 1 - AIDONODE_ENCODING: cbor - experiment_manager_parameters: | - episodes_per_scenario: 1 - episode_length_s: 15.0 - min_episode_length_s: 0 - seed: 42 - physics_dt: 0.05 - max_failures: 2 - agent_in: /fifos/agent-in - agent_out: /fifos/agent-out - sim_in: /fifos/simulator-in - sim_out: /fifos/simulator-out - sm_in: /fifos/scenario_maker-in - sm_out: /fifos/scenario_maker-out - build: - context: ./experiment_manager + evaluator: !include comp_evaluator.yaml + simulator: !include comp_simulator.yaml + solution: !include comp_solution.yaml + scenario_maker: environment: @@ -93,34 +41,4 @@ steps: build: context: ./scenario_maker - simulator: - build: - context: ./simulator - environment: - AIDONODE_DATA_IN: /fifos/simulator-in - AIDONODE_DATA_OUT: fifo:/fifos/simulator-out - AIDONODE_ENCODING: cbor - AIDONODE_CONFIG: | - env_constructor: Simulator - env_parameters: - max_steps: 500001 # we don't want the gym to reset itself - domain_rand: 0 - camera_width: 640 - camera_height: 480 - distortion: true - - solution: - image: SUBMISSION_CONTAINER - environment: - AIDONODE_DATA_IN: /fifos/agent-in - AIDONODE_DATA_OUT: fifo:/fifos/agent-out - AIDONODE_ENCODING: cbor - - - -transitions: - - [START, success, step1-simulation] - - [step1-simulation, success, SUCCESS] - - [step1-simulation, failed, FAILED] - - [step1-simulation, error, ERROR] diff --git a/LFVI.challenge.yaml b/LFVI.challenge.yaml index 62911c9..9c44bf2 100644 --- a/LFVI.challenge.yaml +++ b/LFVI.challenge.yaml @@ -1,87 +1,31 @@ -challenge: aido2_LFVI_r3-v4-z2 +challenge: aido2-LFVI-sim-validation -title: "[AIDO 2] LFVI - Lane following with Vehicles + intersections " -tags: [aido2, aido2-embodied, LF, v4] +title: "LFVI 🚗🚗🚦 - Lane following + Vehicles + Intersections (simulation 👾, validation 🏋) " +tags: [visible, ml-validation, aido2, aido2-embodied, LFVI, v4] description: | protocol: aido2_db18_agent-z2 -date-open: 2018-12-07 -date-close: 2019-12-07 - -roles: {} - -scoring: - scores: - - - name: driven_lanedir_consec_median - short: Traveled distance - description: | - This is the median distance traveled, *along a lane*. - (That is, going in circles will not make this metric increase.) - - This is discretized to tiles. - order: higher-is-better - discretization: 0.62 - - - name: survival_time_median - short: Survival time - description: | - This is the median survival time. The simulation - is terminated when the car goes outside of the road - or it crashes with an obstacle. - order: higher-is-better - discretization: 2.0 - - - name: deviation-center-line_median - short: Lateral deviation - description: | - This is the median lateral deviation from the center line. - order: lower-is-better - discretization: 0.01 - - - name: in-drivable-lane_median - short: Major infractions - description: | - This is the median of the time spent outside of the drivable zones. - For example this penalizes driving in the wrong lane. - order: lower-is-better - discretization: 0.2 - +date-open: 2019-04-10 +date-close: 2019-05-15 +scoring: !include scoring.yaml steps: step1-simulation: title: Simulation step description: "" - timeout: 10 + timeout: 1200 # 20 minutes evaluation_parameters: services: - evaluator: - environment: - DISABLE_CONTRACTS: 1 - AIDONODE_ENCODING: cbor - experiment_manager_parameters: | - episodes_per_scenario: 1 - episode_length_s: 15.0 - min_episode_length_s: 0 - seed: 42 - physics_dt: 0.05 - max_failures: 2 - agent_in: /fifos/agent-in - agent_out: /fifos/agent-out - sim_in: /fifos/simulator-in - sim_out: /fifos/simulator-out - sm_in: /fifos/scenario_maker-in - sm_out: /fifos/scenario_maker-out - build: - context: ./experiment_manager + evaluator: !include comp_evaluator.yaml + simulator: !include comp_simulator.yaml + solution: !include comp_solution.yaml + scenario_maker: environment: - DISABLE_CONTRACTS: 1 - AIDONODE_ENCODING: cbor AIDONODE_DATA_IN: /fifos/scenario_maker-in AIDONODE_DATA_OUT: fifo:/fifos/scenario_maker-out AIDONODE_CONFIG: | @@ -92,34 +36,3 @@ steps: build: context: ./scenario_maker - simulator: - build: - context: ./simulator - environment: - AIDONODE_DATA_IN: /fifos/simulator-in - AIDONODE_DATA_OUT: fifo:/fifos/simulator-out - AIDONODE_ENCODING: cbor - AIDONODE_CONFIG: | - env_constructor: Simulator - env_parameters: - max_steps: 500001 # we don't want the gym to reset itself - domain_rand: 0 - camera_width: 640 - camera_height: 480 - distortion: true - - solution: - image: SUBMISSION_CONTAINER - environment: - AIDONODE_DATA_IN: /fifos/agent-in - AIDONODE_DATA_OUT: fifo:/fifos/agent-out - AIDONODE_ENCODING: cbor - - - - -transitions: - - [START, success, step1-simulation] - - [step1-simulation, success, SUCCESS] - - [step1-simulation, failed, FAILED] - - [step1-simulation, error, ERROR] diff --git a/LFVI_test.challenge.yaml b/LFVI_test.challenge.yaml new file mode 100644 index 0000000..767d707 --- /dev/null +++ b/LFVI_test.challenge.yaml @@ -0,0 +1,36 @@ +challenge: aido2-LFVI-sim-testing + +title: "LFVI 🚗🚗🚦 - Lane following + Vehicles + Intersections (simulation 👾, testing 🥇) " +tags: [visible, ml-testing, aido2, aido2-embodied, LFVI, v4] +description: | + +protocol: aido2_db18_agent-z2 + +date-open: 2019-04-10 +date-close: 2019-05-15 +scoring: !include scoring.yaml + +steps: + step1-simulation: + title: Simulation step + description: "" + + timeout: 1200 # 20 minutes + evaluation_parameters: + services: + evaluator: !include comp_evaluator.yaml + simulator: !include comp_simulator.yaml + solution: !include comp_solution.yaml + + scenario_maker: + environment: + AIDONODE_DATA_IN: /fifos/scenario_maker-in + AIDONODE_DATA_OUT: fifo:/fifos/scenario_maker-out + AIDONODE_CONFIG: | + maps: + - udem1 + scenarios_per_map: 5 + robots_npcs: 4 + build: + context: ./scenario_maker + diff --git a/LFV_test.challenge.yaml b/LFV_test.challenge.yaml new file mode 100644 index 0000000..c8c2fbd --- /dev/null +++ b/LFV_test.challenge.yaml @@ -0,0 +1,44 @@ +challenge: aido2-LFV-sim-testing + +title: "LFV 🚗🚗 - Lane following + Vehicles (simulation 👾, testing 🥇)" +tags: [visible, ml-testing, aido2, aido2-embodied, LFV, v4] +description: | + Lane following challenge. + +protocol: aido2_db18_agent-z2 + +date-open: 2019-04-10 +date-close: 2019-05-15 + +scoring: !include scoring.yaml + + + +steps: + step1-simulation: + title: Simulation step + description: "" + + timeout: 1200 # 20 minutes + evaluation_parameters: + services: + evaluator: !include comp_evaluator.yaml + simulator: !include comp_simulator.yaml + solution: !include comp_solution.yaml + + + scenario_maker: + environment: + DISABLE_CONTRACTS: 1 + AIDONODE_ENCODING: cbor + AIDONODE_DATA_IN: /fifos/scenario_maker-in + AIDONODE_DATA_OUT: fifo:/fifos/scenario_maker-out + AIDONODE_CONFIG: | + maps: + - ETHZ_autolab_technical_track + scenarios_per_map: 5 + robots_npcs: 4 + build: + context: ./scenario_maker + + diff --git a/LF_test.challenge.yaml b/LF_test.challenge.yaml new file mode 100644 index 0000000..a80d23c --- /dev/null +++ b/LF_test.challenge.yaml @@ -0,0 +1,39 @@ +challenge: aido2-LF-sim-testing + +title: "LF 🚗 - Lane following (simulation 👾, testing 🥇)" +tags: [visible, ml-testing, aido2, aido2-embodied, LF, v4] +description: | + Lane following challenge. + +protocol: aido2_db18_agent-z2 + + +date-open: 2019-04-10 +date-close: 2019-05-15 + +scoring: !include scoring.yaml + +steps: + step1-simulation: + title: Simulation step + description: "" + + timeout: 1200 # 20 minutes + evaluation_parameters: + services: + evaluator: !include comp_evaluator.yaml + simulator: !include comp_simulator.yaml + solution: !include comp_solution.yaml + + scenario_maker: + environment: + AIDONODE_DATA_IN: /fifos/scenario_maker-in + AIDONODE_DATA_OUT: fifo:/fifos/scenario_maker-out + AIDONODE_CONFIG: | + maps: + - ETHZ_autolab_technical_track + scenarios_per_map: 5 + robots_npcs: 0 + build: + context: ./scenario_maker + diff --git a/Makefile b/Makefile index f785af7..e2ba3d1 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,17 @@ define-challenge: $(MAKE) define-challenge-LF $(MAKE) define-challenge-LFV $(MAKE) define-challenge-LFVI + $(MAKE) define-challenge-LF_test + $(MAKE) define-challenge-LFV_test + $(MAKE) define-challenge-LFVI_test define-challenge-no-cache: $(MAKE) define-challenge-LF-no-cache $(MAKE) define-challenge-LFV-no-cache $(MAKE) define-challenge-LFVI-no-cache + $(MAKE) define-challenge-LF_test-no-cache + $(MAKE) define-challenge-LFV_test-no-cache + $(MAKE) define-challenge-LFVI_test-no-cache define-challenge-LF: dts challenges define --config LF.challenge.yaml @@ -17,6 +23,15 @@ define-challenge-LFV: define-challenge-LFVI: dts challenges define --config LFVI.challenge.yaml +define-challenge-LF_test: + dts challenges define --config LF_test.challenge.yaml + +define-challenge-LFV_test: + dts challenges define --config LFV_test.challenge.yaml + +define-challenge-LFVI_test: + dts challenges define --config LFVI_test.challenge.yaml + define-challenge-LF-no-cache: dts challenges define --config LF.challenge.yaml --no-cache @@ -27,6 +42,16 @@ define-challenge-LFVI-no-cache: dts challenges define --config LFVI.challenge.yaml --no-cache +define-challenge-LF_test-no-cache: + dts challenges define --config LF_test.challenge.yaml --no-cache + +define-challenge-LFV_test-no-cache: + dts challenges define --config LFV_test.challenge.yaml --no-cache + +define-challenge-LFVI_test-no-cache: + dts challenges define --config LFVI_test.challenge.yaml --no-cache + + test-with-local-repos: docker-compose -f docker-compose-devel.yaml down -v #docker-compose -f docker-compose-devel.yaml build diff --git a/comp_evaluator.yaml b/comp_evaluator.yaml new file mode 100644 index 0000000..6ed1507 --- /dev/null +++ b/comp_evaluator.yaml @@ -0,0 +1,17 @@ + +environment: + experiment_manager_parameters: | + episodes_per_scenario: 1 + episode_length_s: 15.0 + min_episode_length_s: 0.0 + seed: 42 + physics_dt: 0.05 + max_failures: 2 + agent_in: /fifos/agent-in + agent_out: /fifos/agent-out + sim_in: /fifos/simulator-in + sim_out: /fifos/simulator-out + sm_in: /fifos/scenario_maker-in + sm_out: /fifos/scenario_maker-out +build: + context: ./experiment_manager diff --git a/comp_simulator.yaml b/comp_simulator.yaml new file mode 100644 index 0000000..06b3451 --- /dev/null +++ b/comp_simulator.yaml @@ -0,0 +1,13 @@ +build: + context: ./simulator +environment: + AIDONODE_DATA_IN: /fifos/simulator-in + AIDONODE_DATA_OUT: fifo:/fifos/simulator-out + AIDONODE_CONFIG: | + env_constructor: Simulator + env_parameters: + max_steps: 500001 # we don't want the gym to reset itself + domain_rand: 0 + camera_width: 640 + camera_height: 480 + distortion: true diff --git a/comp_solution.yaml b/comp_solution.yaml new file mode 100644 index 0000000..5b73611 --- /dev/null +++ b/comp_solution.yaml @@ -0,0 +1,5 @@ + +image: SUBMISSION_CONTAINER +environment: + AIDONODE_DATA_IN: /fifos/agent-in + AIDONODE_DATA_OUT: fifo:/fifos/agent-out diff --git a/minimal_agent/submission.yaml b/minimal_agent/submission.yaml index 3d9060a..518cf65 100644 --- a/minimal_agent/submission.yaml +++ b/minimal_agent/submission.yaml @@ -1,13 +1,10 @@ -# challenge name -challenge: -- aido2_LF_r3-v4-z2 -- aido2_LFV_r3-v4-z2 -- aido2_LFVI_r3-v4-z2 +# challenge name(s); defaults to all compatible challenges +# challenge: [c1,c2] # conventions + format protocol: aido2_db18_agent-z2 # any user label - displayed in "challenges list" -user-label: Minimal agent (Python 3) +user-label: minimal_agent (Python 3) # any JSON payload user-payload: {} diff --git a/minimal_agent_python2/submission.yaml b/minimal_agent_python2/submission.yaml index fe7b9fd..b116e56 100644 --- a/minimal_agent_python2/submission.yaml +++ b/minimal_agent_python2/submission.yaml @@ -1,13 +1,10 @@ -# challenge name -challenge: -- aido2_LF_r3-v4-z2 -- aido2_LFV_r3-v4-z2 -- aido2_LFVI_r3-v4-z2 +# challenge name(s); defaults to all compatible challenges +# challenge: [c1,c2] # conventions + format protocol: aido2_db18_agent-z2 # any user label - displayed in "challenges list" -user-label: Minimal agent (Python 2) +user-label: minimal_agent_python2 (Python 2) # any JSON payload user-payload: {} diff --git a/scoring.yaml b/scoring.yaml new file mode 100644 index 0000000..1fc2842 --- /dev/null +++ b/scoring.yaml @@ -0,0 +1,35 @@ +scores: + + - name: driven_lanedir_consec_median + short: Traveled distance + description: | + This is the median distance traveled, *along a lane*. + (That is, going in circles will not make this metric increase.) + + This is discretized to tiles. + order: higher-is-better + discretization: 0.62 + + - name: survival_time_median + short: Survival time + description: | + This is the median survival time. The simulation + is terminated when the car goes outside of the road + or it crashes with an obstacle. + order: higher-is-better + discretization: 2.0 + + - name: deviation-center-line_median + short: Lateral deviation + description: | + This is the median lateral deviation from the center line. + order: lower-is-better + discretization: 0.01 + + - name: in-drivable-lane_median + short: Major infractions + description: | + This is the median of the time spent outside of the drivable zones. + For example this penalizes driving in the wrong lane. + order: lower-is-better + discretization: 0.2