diff --git a/tensor2tensor/rl/gym_utils.py b/tensor2tensor/rl/gym_utils.py
index 81b9df7ed..5d0e33c5d 100644
--- a/tensor2tensor/rl/gym_utils.py
+++ b/tensor2tensor/rl/gym_utils.py
@@ -343,7 +343,7 @@ def make_gym_env(name,
                          output_dtype, num_actions)
 
 
-def register_gym_env(class_entry_point, version="v0", kwargs=None):
+def register_gym_env(class_entry_point, version="v0", **kwargs):
   """Registers the class in Gym and returns the registered name and the env."""
 
   split_on_colon = class_entry_point.split(":")