LFV_test.challenge.yaml

challenge: aido3-LFV-sim-testing

title: "LFV 🚗🚗 - Lane following + Vehicles (simulation 👾, testing 🥇)"
tags: [visible, ml-testing, aido3, aido3-embodied, LFV, v4, simulation]
description: |
  Lane following challenge.

protocol: aido2_db18_agent-z2

date-open: 2019-10-10T00:00Z
date-close: 2019-12-12T13:00Z


scoring: !include scoring.yaml


dependencies:
  aido3-LFV-sim-validation:
    description: The submission must first pass the testing.
    min_threshold: 2.0
    scores:
      good_enough:
        description: Must have moved at least 20cm.
        importance: 1.0
        score_name: driven_lanedir_consec_median
        score_min: 0.2
        score_max: 10000.0
    comparisons:
      better-than-bea-straight:
        description: Must be do better than the straight solution.
        importance: 1.0
        username: "1639"
        sub_label: 'straight'

steps:
  step1-simulation:
    title: Simulation step
    description: ""
    features_required:
      compute_sims: 1 # computation machine


    timeout: 3600 # 1 hour
    evaluation_parameters:
      services:
        evaluator: !include comp_evaluator.yaml
        simulator: !include comp_simulator.yaml
        solution: !include comp_solution.yaml


        scenario_maker:
          environment:
            DISABLE_CONTRACTS: 1
            AIDONODE_ENCODING: cbor
            AIDONODE_DATA_IN: /fifos/scenario_maker-in
            AIDONODE_DATA_OUT: fifo:/fifos/scenario_maker-out
            AIDONODE_CONFIG: |
              maps:
              - ETHZ_autolab_technical_track
              scenarios_per_map: 15
              robots_npcs: 4
          build:
            context: ./scenario_maker