Merge pull request #635 from JdeRobot/issue-633

Added CARLA examples for GUI and non-GUI pipelines

.gitignore

@@ -68,6 +68,7 @@
 *.max
 
 *.h5
+*.pth
 
 ### Python ###
 # Byte-compiled / optimized / DLL files

README.md

@@ -3,17 +3,57 @@
 [![Publish cuda 11 Docker image](https://github.com/JdeRobot/BehaviorMetrics/actions/workflows/generate_docker_cuda_11.yml/badge.svg)](https://github.com/JdeRobot/BehaviorMetrics/actions/workflows/generate_docker_cuda_11.yml)
 # Behavior Metrics
 
-This project presents different approaches to the follow-the-line exercise but using artificial intelligence to complete the circuits. The solutions presented are:
-- Using classification networks.
-- Using regression networks.
-- Using reinforcement learning.
-- Solution for real robots.
+This software tool provides evaluation capabilities for autonomous driving solutions using simulation. 
+We provide a series of quantitative metrics for the evaluation of autonomous driving solutions with support for two simulators, [CARLA](https://carla.org/) (main supported simulator) and [gazebo](https://gazebosim.org/home) (partial support).
+Currently supported tasks include:
 
-For more information about the project you can consult the [website of Behavior Metrics](https://jderobot.github.io/BehaviorMetrics/). 
+* Follow-lane
+* Driving in traffic
+* Navigation
+
+Each task comes with its own custom evaluation metrics that can help compare autonomous driving solutions.
+The main component of the ego vehicle is the brain, which receives sensor data, manipulates it, and generates robot control commands based on it. 
+The inner part of the brain can be controlled by an end-to-end model, written in Tensorflow or PyTorch, a reinforcement learning policy, or even an explicitly programmed policy.
+
+The software provides two main pipelines, a graphical user interface (GUI) and a headless mode (scripted). 
+The first one is intended for testing one brain+model at a time and debugging it visually while the headless mode is intended for running lots of experiments at the same time for comparison of a batch of brain+models in different scenarios.
+
+<img src="./assets/behavior_metrics_full_architecture.png" alt="architecture"/>
+
+### Installation
+
+For more information about the project and how to install it, you can consult the [website of Behavior Metrics](https://jderobot.github.io/BehaviorMetrics/). 
+
+### Examples
+
+We provide examples for the follow-lane task using CARLA:
+
+* For an example of a robot brain using a Tensorflow model for control with GUI pipeline, run:
+
+```
+python3 driver_carla.py -c configs/CARLA/default_carla_tensorflow.yml -g
+```
+
+* For an example of a robot brain using a PyTorch model for control with GUI pipeline, run:
+
+```
+python3 driver_carla.py -c configs/CARLA/default_carla_torch.yml -g
+```
+
+* For an example of an explicitly programmed robot brain with GUI pipeline, run:
+
+```
+python3 driver_carla.py -c configs/CARLA/default_carla.yml -g
+```
+
+* For an example of the headless pipeline, run:
+
+```
+python3 driver_carla.py -c configs/CARLA/default_carla_multiple.yml -s
+```
 
 ### Contributing to the project
 
 If you want to contribute, please first check out [CONTRIBUTING.md](CONTRIBUTING.md) section.
 
 <img src="https://jderobot.github.io/assets/images/projects/neural_behavior/autonomous.jpeg" alt="config" style="zoom:20%;" />
-<img src="./assets/behavior_metrics_full_architecture.png" alt="architecture"/>

behavior_metrics/brains/CARLA/brain_carla_slow_and_turn.py

@@ -65,7 +65,9 @@ def execute(self):
         image_2 = self.camera_2.getImage().data
         image_3 = self.camera_3.getImage().data
 
-        self.motors.sendThrottle(1)
+        self.motors.sendThrottle(0.5)
+        self.motors.sendSteer(0.05)
+
         self.update_frame('frame_0', image)
         self.update_frame('frame_1', image_1)
         self.update_frame('frame_2', image_2)

behavior_metrics/brains/CARLA/brain_carla_subjective_vision_deep_learning_previous_v.py

@@ -16,7 +16,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/pytorch/brain_carla_bird_eye_deep_learning_torch.py

@@ -1,5 +1,5 @@
 from PIL import Image
-from brains.f1.torch_utils.pilotnet import PilotNet
+from brains.CARLA.pytorch.utils.pilotnet import PilotNet
 from utils.constants import PRETRAINED_MODELS_DIR, ROOT_PATH
 from os import path
 from albumentations import (
@@ -17,7 +17,7 @@
 import math
 import carla
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 FLOAT = torch.FloatTensor
 
 class Brain:

behavior_metrics/brains/CARLA/pytorch/brain_carla_bird_eye_deep_learning_torch_optimized.py

@@ -1,5 +1,5 @@
 from PIL import Image
-from brains.f1.torch_utils.pilotnet import PilotNet
+from brains.CARLA.pytorch.utils.pilotnet import PilotNet
 from utils.constants import PRETRAINED_MODELS_DIR, ROOT_PATH
 from os import path
 from albumentations import (
@@ -17,7 +17,7 @@
 import math
 import carla
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 FLOAT = torch.FloatTensor
 
 class Brain:

behavior_metrics/brains/CARLA/pytorch/brain_carla_bird_eye_deep_learning_torch_tensorrt_float.py

@@ -1,5 +1,5 @@
 from PIL import Image
-from brains.f1.torch_utils.pilotnet import PilotNet
+from brains.CARLA.pytorch.utils.pilotnet import PilotNet
 from utils.constants import PRETRAINED_MODELS_DIR, ROOT_PATH
 from os import path
 from albumentations import (
@@ -17,7 +17,7 @@
 import math
 import carla
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 FLOAT = torch.FloatTensor
 
 class Brain:

behavior_metrics/brains/CARLA/pytorch/brain_carla_bird_eye_deep_learning_torch_tensorrt_int_8.py

@@ -1,5 +1,5 @@
 from PIL import Image
-from brains.f1.torch_utils.pilotnet import PilotNet
+from brains.CARLA.pytorch.utils.pilotnet import PilotNet
 from utils.constants import PRETRAINED_MODELS_DIR, ROOT_PATH
 from os import path
 from albumentations import (
@@ -17,7 +17,7 @@
 import math
 import carla
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 FLOAT = torch.FloatTensor
 
 class Brain:

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError
@@ -136,6 +136,7 @@ def execute(self):
         self.update_pose(self.pose.getPose3d())
 
         image_shape=(66, 200)
+        #image_shape=(50, 150)
         img_base = cv2.resize(bird_eye_view_1, image_shape)
 
         AUGMENTATIONS_TEST = Compose([

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_70_km_h.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_V_MAX_30.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_broken_input.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_broken_input_2.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_previous_v.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_previous_v_70_km_h.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_previous_v_broken_input.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_previous_v_broken_input_2.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_tensor_rt.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_tf_lite.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_x3_previous_v_t_t-4_t-9.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_x3_previous_v_t_t-4_t-9_70_km_h.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_x3_previous_v_t_t-4_t-9_broken_input.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_x3_previous_v_t_t-4_t-9_broken_input_2.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_x3_t_t-4_t-9.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_x3_t_t-4_t-9_70_km_h.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_x3_t_t-4_t-9_V_MAX_30.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_x3_t_t-4_t-9_broken_input.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/CARLA/tensorflow/brain_carla_bird_eye_deep_learning_x3_t_t-4_t-9_broken_input_2.py

@@ -15,7 +15,7 @@
 from utils.logger import logger
 from traceback import print_exc
 
-PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'carla_tf_models/'
+PRETRAINED_MODELS = ROOT_PATH + '/' + PRETRAINED_MODELS_DIR + 'CARLA/'
 
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.framework.errors_impl import UnimplementedError

behavior_metrics/brains/brains_handler.py

@@ -34,9 +34,13 @@ def __init__(self, sensors, actuators, brain_path, controller, model=None, confi
     def load_brain(self, path, model=None):
 
         path_split = path.split("/")
-        robot_type = path_split[-2]
+        robot_type = path_split[1]
         module_name = path_split[-1][:-3]  # removing .py extension
-        import_name = 'brains.' + robot_type + '.' + module_name
+        if len(path_split) == 4:
+            framework = path_split[2]
+            import_name = 'brains.' + robot_type + '.' + framework + '.' + module_name
+        else:
+            import_name = 'brains.' + robot_type + '.' + module_name
 
         if robot_type == 'CARLA':
             module = importlib.import_module(import_name)

behavior_metrics/brains/f1/brain_f1_follow_line_dqn.py → behavior_metrics/brains/gazebo/f1/brain_f1_follow_line_dqn.py

@@ -1,7 +1,7 @@
 
 import tensorflow as tf
 from gym.envs.registration import register
-from brains.f1.rl_utils.inference import InferencerWrapper
+from brains.gazebo.f1.rl_utils.inference import InferencerWrapper
 import yaml
 import gym
 import numpy as np
@@ -11,7 +11,7 @@
 if 'F1Env-v0' not in gym.envs.registry.env_specs:
     gym.envs.register(
         id='F1Env-v0',
-        entry_point='brains.f1.rl_utils.models:F1Env',
+        entry_point='brains.gazebo.f1.rl_utils.models:F1Env',
         # More arguments here
     )
 else:
@@ -154,7 +154,7 @@ def __init__(self, sensors, actuators, handler, config=None):
             'algorithm': 'dqn',
             'environment': 'simple',
             'agent': 'f1',
-            'filename': 'brains/f1/config/config_inference_followline_dqn_f1_gazebo.yaml'
+            'filename': 'brains/gazebo/f1/config/config_inference_followline_dqn_f1_gazebo.yaml'
         }
 
         f = open(args['filename'], "r")