Merge branch 'master' into main

.gitignore

@@ -0,0 +1,167 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+data/
+ckpts/
+data
+ckpts
+out/
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+.vscode
+outputs/
+submission/
+checkpoints/

README.md

@@ -1 +1,117 @@
-# DeMo
+# [NeurIPS 2024] DeMo: Decoupling Motion Forecasting into Directional Intentions and Dynamic States
+### [[Paper]](https://arxiv.org/abs/XXXX)
+
+> [**DeMo: Decoupling Motion Forecasting into Directional Intentions and Dynamic States**](https://arxiv.org/abs/XXXX),            
+> [Bozhou Zhang](https://zbozhou.github.io/), Nan Song, [Li Zhang](https://lzrobots.github.io)  
+> **Fudan University**
+
+## 🚗 Abstract
+Accurate motion forecasting for traffic agents is crucial for ensuring the safety and efficiency of autonomous driving systems in dynamically changing environments. Mainstream methods adopt a one-query-one-trajectory paradigm, where each query corresponds to a unique trajectory for predicting multi-modal trajectories. While straightforward and effective, the absence of detailed representation of future trajectories may yield suboptimal outcomes, given that the agent states dynamically evolve over time. To address this problem, we introduce DeMo, a framework that decouples multi-modal trajectory queries into two types: mode queries capturing distinct directional intentions and state queries tracking the agent's dynamic states over time. By leveraging this format, we separately optimize the multi-modality and dynamic evolutionary properties of trajectories. Subsequently, the mode and state queries are integrated to obtain a comprehensive and detailed representation of the trajectories. To achieve these operations, we additionally introduce combined Attention and Mamba techniques for global information aggregation and state sequence modeling, leveraging their respective strengths. Extensive experiments on both the Argoverse 2 and nuScenes benchmarks demonstrate that our DeMo achieves state-of-the-art performance in motion forecasting. 
+
+## 🎞️ Pipeline
+<div align="center">
+  <img src="assets/main.jpg"/>
+</div><br/>
+
+## 🛠️ Get started
+
+### Set up a new virtual environment
+```
+conda create -n DeMo python=3.10
+conda activate DeMo
+```
+
+### Install dependency packpages
+```
+pip install torch==2.1.1 torchvision==0.16.1 torchaudio==2.1.1 --index-url https://download.pytorch.org/whl/cu118
+pip install -r ./requirements.txt
+pip install av2
+```
+
+### Install Mamba
+- We follow the settings outlined in [VideoMamba](https://github.com/OpenGVLab/VideoMamba).
+```
+git clone [email protected]:OpenGVLab/VideoMamba.git
+cd VideoMamba
+pip install -e causal-conv1d
+pip install -e mamba
+```
+
+### Some packages may be useful
+```
+pip install tensorboard
+pip install torch-scatter -f https://data.pyg.org/whl/torch-2.1.1+cu118.html
+pip install protobuf==3.20.3
+```
+
+## 🕹️ Prepare the data
+### Setup [Argoverse 2 Motion Forecasting Dataset](https://www.argoverse.org/av2.html)
+```
+data_root
+    ├── train
+    │   ├── 0000b0f9-99f9-4a1f-a231-5be9e4c523f7
+    │   ├── 0000b6ab-e100-4f6b-aee8-b520b57c0530
+    │   ├── ...
+    ├── val
+    │   ├── 00010486-9a07-48ae-b493-cf4545855937
+    │   ├── 00062a32-8d6d-4449-9948-6fedac67bfcd
+    │   ├── ...
+    ├── test
+    │   ├── 0000b329-f890-4c2b-93f2-7e2413d4ca5b
+    │   ├── 0008c251-e9b0-4708-b762-b15cb6effc27
+    │   ├── ...
+```
+
+### Preprocess
+```
+python preprocess.py --data_root=/path/to/data_root -p
+```
+
+### The structure of the dataset after processing
+```
+└── data
+    └── DeMo_processed
+        ├── train
+        ├── val
+        └── test
+```
+
+## 🔥 Training and testing
+```
+# Train
+python train.py 
+
+# Val, remember to change the checkpoint to your own in eval.py
+python eval.py
+
+# Test for submission
+python eval.py gpus=1 test=true
+```
+
+## ⭐ Results and checkpoints
+- We provide two versions of the models: `DeMo` for [DeMo](https://github.com/fudan-zvg/DeMo) itself, and `DeMo+RealMotion`, which integrates [DeMo](https://github.com/fudan-zvg/DeMo) with our other work, [RealMotion](). You can select which model to use by adjusting the first 3 rows in [config.yaml]().
+
+| Models | minADE1 | minFDE1 | minADE6 | minFDE6 |
+| :- | :-: | :-: | :-: | :-: |
+| [DeMo](https://drive.google.com/file/d/1xqj8T5M2cczIZU26poseAQri6VE6v9a0/view?usp=drive_link)   |  1.578  |  3.961  |  0.645  |  1.247  |
+| [DeMo+RealMotion](https://drive.google.com/file/d/131pcHXP-vLcyypZWn6Es7n6TT8bxs4rN/view?usp=drive_link) |  1.478  |  3.728  |  0.607  |  1.186  |
+
+### Qualitative Results
+<div align="center">
+  <img src="assets/visual.jpg"/>
+</div><br/>
+
+## 📜 BibTeX
+```bibtex
+@article{zhang2024demo,
+  title={DeMo: Decoupling Motion Forecasting into Directional Intentions and Dynamic States},
+  author={Zhang, Bozhou and Song, Nan and Zhang, Li},
+  journal={arXiv preprint arXiv:XXXX.XXXX},
+  year={2024},
+}
+```
+
+## ❤️ Acknowledgements
+ - [VideoMamba](https://github.com/OpenGVLab/VideoMamba)
+ - [Forecast-MAE](https://github.com/jchengai/forecast-mae)
+ - [StreamPETR](https://github.com/exiawsh/StreamPETR)

conf/config.yaml

@@ -0,0 +1,40 @@
+defaults:
+  - datamodule: av2  # 'av2' for 'DeMo', 'av2_stream' for 'DeMo+RealMotion'
+  - model: model_forecast  # 'model_forecast' for 'DeMo', 'stream_model_forecast' for 'DeMo+RealMotion'
+# eg: For 'DeMo'
+# defaults:
+#   - datamodule: av2
+#   - model: model_forecast
+
+hydra:
+  run:
+    dir: outputs/${output}/${now:%Y%m%d-%H%M%S}
+
+seed: 2024
+monitor: val_minFDE6
+save_top_k: 10
+
+data_root:
+checkpoint:
+pretrained_weights:
+output: change_${datamodule.name}_${datamodule.phase}_${model.name}_${model.phase}
+
+# trainer
+num_workers: 4
+gpus: 8  # when test, only use 1 gpu; when train/val, use 8 gpus
+sync_bn: true
+batch_size: 16
+epochs: 60
+warmup_epochs: 10
+
+# optimizer
+lr: 0.003  
+weight_decay: 1e-2  
+gradient_clip_val: 5
+gradient_clip_algorithm: norm
+
+limit_train_batches:
+limit_val_batches:
+limit_test_batches:
+log_model: all
+test: false  # false for val, true for test

conf/datamodule/av2.yaml

@@ -0,0 +1,18 @@
+name: av2
+phase: default
+
+target:
+  _target_: src.datamodule.av2_datamodule.Av2DataModule
+
+  data_root: 'data/DeMo_processed'
+  train_batch_size: ${batch_size}
+  val_batch_size: ${batch_size}
+  test_batch_size: ${batch_size}
+  shuffle: True
+  num_workers: ${num_workers}
+  pin_memory: True
+  dataset:
+    num_historical_steps: 50
+    sequence_origins: [50]
+    radius: 150.0
+    train_mode: 'only_focal'

conf/datamodule/av2_stream.yaml

@@ -0,0 +1,18 @@
+name: av2-stream
+phase: 3frame-30his
+
+target:
+  _target_: src.datamodule.av2_datamodule.Av2DataModule
+
+  data_root: 'data/DeMo_processed'
+  train_batch_size: ${batch_size}
+  val_batch_size: ${batch_size}
+  test_batch_size: ${batch_size}
+  shuffle: True
+  num_workers: ${num_workers}
+  pin_memory: True
+  dataset:
+    num_historical_steps: 30
+    sequence_origins: [30, 40, 50]
+    radius: 150.0
+    train_mode: 'only_focal'

conf/model/model_forecast.yaml

@@ -0,0 +1,18 @@
+name: baseline
+phase: default
+
+target:
+  _target_: src.model.trainer_forecast.Trainer
+  model:
+    type: 'ModelForecast'
+    embed_dim: 128
+    future_steps: 60
+    num_heads: 8
+    mlp_ratio: 4.0
+    qkv_bias: False
+    drop_path: 0.2
+  pretrained_weights: ${pretrained_weights}
+  lr: ${lr}
+  weight_decay: ${weight_decay}
+  epochs: ${epochs}
+  warmup_epochs: ${warmup_epochs}