diff --git a/minigrid/__init__.py b/minigrid/__init__.py index 154fb611d..155e1413a 100644 --- a/minigrid/__init__.py +++ b/minigrid/__init__.py @@ -1133,3 +1133,16 @@ def register_minigrid_envs(): id="BabyAI-BossLevelNoUnlock-v0", entry_point="minigrid.envs.babyai:BossLevelNoUnlock", ) + + # BabyAI - Language based levels - Level_MixedTrainLocal and Level_MixedTestLocal + # ---------------------------------------- + + register( + id="BabyAI-MixedTrainLocal-v0", + entry_point="minigrid.envs.babyai:Level_MixedTrainLocal", + ) + + register( + id="BabyAI-MixedTestLocal-v0", + entry_point="minigrid.envs.babyai:Level_MixedTestLocal", + ) diff --git a/minigrid/envs/babyai/__init__.py b/minigrid/envs/babyai/__init__.py index fb39fbf48..45fe2173a 100644 --- a/minigrid/envs/babyai/__init__.py +++ b/minigrid/envs/babyai/__init__.py @@ -13,6 +13,10 @@ GoToRedBlueBall, GoToSeq, ) +from minigrid.envs.babyai.mixed_seq_levels import ( + Level_MixedTestLocal, + Level_MixedTrainLocal, +) from minigrid.envs.babyai.open import ( Open, OpenDoor, diff --git a/minigrid/envs/babyai/core/levelgen.py b/minigrid/envs/babyai/core/levelgen.py index fc30b9894..fe744074d 100644 --- a/minigrid/envs/babyai/core/levelgen.py +++ b/minigrid/envs/babyai/core/levelgen.py @@ -83,7 +83,7 @@ def gen_mission(self): action_kinds=self.action_kinds, instr_kinds=self.instr_kinds ) - def add_locked_room(self): + def add_locked_room(self, color=None): # Until we've successfully added a locked room while True: i = self._rand_int(0, self.num_cols) @@ -95,7 +95,10 @@ def add_locked_room(self): if self.locked_room.neighbors[door_idx] is None: continue - door, _ = self.add_door(i, j, door_idx, locked=True) + if color is not None: + door, _ = self.add_door(i, j, door_idx, color=color, locked=True) + else: + door, _ = self.add_door(i, j, door_idx, locked=True) # Done adding locked room break diff --git a/minigrid/envs/babyai/mixed_seq_levels.py b/minigrid/envs/babyai/mixed_seq_levels.py new file mode 100644 index 000000000..5870a2a32 --- /dev/null +++ b/minigrid/envs/babyai/mixed_seq_levels.py @@ -0,0 +1,1231 @@ +""" +Copied and adapted from https://github.com/flowersteam/Grounding_LLMs_with_online_RL +""" + +from __future__ import annotations + +from minigrid.envs.babyai.core.levelgen import ( + GoToInstr, + LevelGen, + PickupInstr, + PutNextInstr, +) +from minigrid.envs.babyai.core.verifier import ( + AfterInstr, + BeforeInstr, + ObjDesc, + OpenInstr, +) + + +class Level_MixedTrainLocal(LevelGen): + """ + Union of all instructions from PutNext, Open, Goto and PickUp. + The agent does not need to move objects around. + When the task is Open there are 2 rooms and the door in between is locked. + For the other instructions there is only one room. + Sequence of action are possible. + + In order to test generalisation we do not give to the agent the instructions containing: + - yellow box + - red door/key + - green ball + - grey door + - seq is restricted to pick up A then/before go to B (for memory issue our agent only used the past 3 observations) + + At test time we release the 3 first previous constraints, and we add to seq + pick up A then/before pick up B + + Competencies: Unlock, GoTo, PickUp, PutNext, Seq + """ + + def __init__( + self, + room_size=8, + num_rows=1, + num_cols=1, + num_dists=8, + instr_kinds=["action", "seq1"], + locations=False, + unblocking=False, + implicit_unlock=False, + **kwargs, + ): + action = self._rand_elem( + ["goto", "pickup", "open", "putnext", "pick up seq go to"] + ) + if action == "open": + num_cols = 2 + num_rows = 1 + # We add many distractors to increase the probability + # of ambiguous locations within the same room + super().__init__( + room_size=room_size, + num_rows=num_rows, + num_cols=num_cols, + num_dists=num_dists, + action_kinds=[action], + instr_kinds=instr_kinds, + locations=locations, + unblocking=unblocking, + implicit_unlock=implicit_unlock, + **kwargs, + ) + + # ['goto', 'pickup', 'open', 'putnext', 'pick up seq go to'], + def gen_mission(self): + action = self._rand_elem(self.action_kinds) + mission_accepted = False + all_objects_reachable = False + if action == "open": + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + color_door = self._rand_elem( + ["yellow", "green", "blue", "purple"] + ) # red and grey excluded + self.add_locked_room(color_door) + self.connect_all() + + for j in range(self.num_rows): + for i in range(self.num_cols): + if self.get_room(i, j) is not self.locked_room: + self.add_distractors( + i, j, num_distractors=self.num_dists, all_unique=False + ) + + # The agent must be placed after all the object to respect constraints + while True: + self.place_agent() + start_room = self.room_from_pos(*self.agent_pos) + # Ensure that we are not placing the agent in the locked room + if start_room is self.locked_room: + continue + break + + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + + color_in_instr = self._rand_elem([None, color_door]) + + desc = ObjDesc("door", color_in_instr) + self.instrs = OpenInstr(desc) + + mission_accepted = not (self.exclude_substrings()) + + """if color_in_instr is None and mission_accepted and all_objects_reachable: + print(color_door)""" + + elif action == "goto": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 1, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj = self._rand_elem(objs) + self.instrs = GoToInstr(ObjDesc(obj.type, obj.color)) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "pickup": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 1, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj = self._rand_elem(objs) + while str(obj.type) == "door": + obj = self._rand_elem(objs) + self.instrs = PickupInstr(ObjDesc(obj.type, obj.color)) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "putnext": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 2, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj_1 = self._rand_elem(objs) + while str(obj_1.type) == "door": + obj_1 = self._rand_elem(objs) + desc1 = ObjDesc(obj_1.type, obj_1.color) + obj_2 = self._rand_elem(objs) + if obj_1.type == obj_2.type and obj_1.color == obj_2.color: + obj1s, poss = desc1.find_matching_objs(self) + if len(obj1s) < 2: + # if obj_1 is the only object with this description obj_2 has to be different + while obj_1.type == obj_2.type and obj_1.color == obj_2.color: + obj_2 = self._rand_elem(objs) + desc2 = ObjDesc(obj_2.type, obj_2.color) + self.instrs = PutNextInstr(desc1, desc2) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "pick up seq go to": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 2, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj_a = self._rand_elem(objs) + while str(obj_a.type) == "door": + obj_a = self._rand_elem(objs) + instr_a = PickupInstr(ObjDesc(obj_a.type, obj_a.color)) + obj_b = self._rand_elem(objs) + if obj_a.type == obj_b.type and obj_a.color == obj_b.color: + desc = ObjDesc(obj_a.type, obj_a.color) + objas, poss = desc.find_matching_objs(self) + if len(objas) < 2: + # if obj_a is the only object with this description obj_b has to be different + while obj_a.type == obj_b.type and obj_a.color == obj_b.color: + obj_b = self._rand_elem(objs) + instr_b = GoToInstr(ObjDesc(obj_b.type, obj_b.color)) + + type_instr = self._rand_elem(["Before", "After"]) + + if type_instr == "Before": + self.instrs = BeforeInstr(instr_a, instr_b) + else: + self.instrs = AfterInstr(instr_b, instr_a) + + mission_accepted = not (self.exclude_substrings()) + + def exclude_substrings(self): + # True if contains excluded substring + list_exclude_combinaison = [ + "yellow box", + "red key", + "red door", + "green ball", + "grey door", + ] + + for sub_str in list_exclude_combinaison: + str = self.instrs.surface(self) + if sub_str in self.instrs.surface(self): + return True + return False + + def _regen_grid(self): + # Create the grid + self.grid.grid = [None] * self.width * self.height + + # For each row of rooms + for j in range(0, self.num_rows): + row = [] + + # For each column of rooms + for i in range(0, self.num_cols): + room = self.get_room(i, j) + # suppress doors and objects + room.doors = [None] * 4 + room.door_pos = [None] * 4 + room.neighbors = [None] * 4 + room.locked = False + room.objs = [] + row.append(room) + + # Generate the walls for this room + self.grid.wall_rect(*room.top, *room.size) + + self.room_grid.append(row) + + # For each row of rooms + for j in range(0, self.num_rows): + # For each column of rooms + for i in range(0, self.num_cols): + room = self.room_grid[j][i] + + x_l, y_l = (room.top[0] + 1, room.top[1] + 1) + x_m, y_m = ( + room.top[0] + room.size[0] - 1, + room.top[1] + room.size[1] - 1, + ) + + # Door positions, order is right, down, left, up + if i < self.num_cols - 1: + room.neighbors[0] = self.room_grid[j][i + 1] + room.door_pos[0] = (x_m, self._rand_int(y_l, y_m)) + if j < self.num_rows - 1: + room.neighbors[1] = self.room_grid[j + 1][i] + room.door_pos[1] = (self._rand_int(x_l, x_m), y_m) + if i > 0: + room.neighbors[2] = self.room_grid[j][i - 1] + room.door_pos[2] = room.neighbors[2].door_pos[0] + if j > 0: + room.neighbors[3] = self.room_grid[j - 1][i] + room.door_pos[3] = room.neighbors[3].door_pos[1] + + # The agent starts in the middle, facing right + self.agent_pos = ( + (self.num_cols // 2) * (self.room_size - 1) + (self.room_size // 2), + (self.num_rows // 2) * (self.room_size - 1) + (self.room_size // 2), + ) + self.agent_dir = 0 + + +class Level_MixedTestLocal(LevelGen): + """ + Union of all instructions from PutNext, Open, Goto and PickUp. + The agent does not need to move objects around. + When the task is Open there are 2 rooms and the door in between is locked. + For the other instructions there is only one room. + Sequence of action are possible. + + In order to test generalisation we only give to the agent the instructions containing: + - yellow box + - red door/key + - green ball + - grey door + - seq is restricted to pick up A then/before go to B with A and B among the previous adj-noun pairs + (for memory issue our agent only used the past 3 observations) + + Competencies: Unlock, GoTo, PickUp, PutNext, Seq + """ + + def __init__( + self, + room_size=8, + num_rows=1, + num_cols=1, + num_dists=8, + instr_kinds=["action", "seq1"], + locations=False, + unblocking=False, + implicit_unlock=False, + **kwargs, + ): + action = self._rand_elem( + ["goto", "pickup", "open", "putnext", "pick up seq go to"] + ) + if action == "open": + num_cols = 2 + num_rows = 1 + # We add many distractors to increase the probability + # of ambiguous locations within the same room + super().__init__( + room_size=room_size, + num_rows=num_rows, + num_cols=num_cols, + num_dists=num_dists, + action_kinds=[action], + instr_kinds=instr_kinds, + locations=locations, + unblocking=unblocking, + implicit_unlock=implicit_unlock, + **kwargs, + ) + + def gen_mission(self): + action = self._rand_elem(self.action_kinds) + mission_accepted = False + all_objects_reachable = False + if action == "open": + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + color_door = self._rand_elem( + ["red", "grey"] + ) # only red and grey doors at test time + self.add_locked_room(color_door) + self.connect_all() + + for j in range(self.num_rows): + for i in range(self.num_cols): + if self.get_room(i, j) is not self.locked_room: + self.add_distractors( + i, j, num_distractors=self.num_dists, all_unique=False + ) + + # The agent must be placed after all the object to respect constraints + while True: + self.place_agent() + start_room = self.room_from_pos(*self.agent_pos) + # Ensure that we are not placing the agent in the locked room + if start_room is self.locked_room: + continue + break + + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + + desc = ObjDesc("door", color_door) + self.instrs = OpenInstr(desc) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "goto": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 1, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj = self._rand_elem(objs) + self.instrs = GoToInstr(ObjDesc(obj.type, obj.color)) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "pickup": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 1, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj = self._rand_elem(objs) + while str(obj.type) == "door": + obj = self._rand_elem(objs) + self.instrs = PickupInstr(ObjDesc(obj.type, obj.color)) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "putnext": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 2, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj_1 = self._rand_elem(objs) + while str(obj_1.type) == "door": + obj_1 = self._rand_elem(objs) + desc1 = ObjDesc(obj_1.type, obj_1.color) + obj_2 = self._rand_elem(objs) + if obj_1.type == obj_2.type and obj_1.color == obj_2.color: + obj1s, poss = desc1.find_matching_objs(self) + if len(obj1s) < 2: + # if obj_1 is the only object with this description obj_2 has to be different + while obj_1.type == obj_2.type and obj_1.color == obj_2.color: + obj_2 = self._rand_elem(objs) + desc2 = ObjDesc(obj_2.type, obj_2.color) + self.instrs = PutNextInstr(desc1, desc2) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "pick up seq go to": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 2, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj_a = self._rand_elem(objs) + while str(obj_a.type) == "door": + obj_a = self._rand_elem(objs) + instr_a = PickupInstr(ObjDesc(obj_a.type, obj_a.color)) + obj_b = self._rand_elem(objs) + if obj_a.type == obj_b.type and obj_a.color == obj_b.color: + desc = ObjDesc(obj_a.type, obj_a.color) + objas, poss = desc.find_matching_objs(self) + if len(objas) < 2: + # if obj_a is the only object with this description obj_b has to be different + while obj_a.type == obj_b.type and obj_a.color == obj_b.color: + obj_b = self._rand_elem(objs) + instr_b = GoToInstr(ObjDesc(obj_b.type, obj_b.color)) + + type_instr = self._rand_elem(["Before", "After"]) + + if type_instr == "Before": + self.instrs = BeforeInstr(instr_a, instr_b) + else: + self.instrs = AfterInstr(instr_b, instr_a) + + mission_accepted = not (self.exclude_substrings()) + + def exclude_substrings(self): + # True if contains excluded substring + list_exclude_combinaison = [ + "yellow key", + "yellow ball", + "yellow door", + "red box", + "red ball", + "green box", + "green key", + "green door", + "grey box", + "grey key", + "grey ball", + "blue box", + "blue key", + "blue ball", + "blue door", + "purple box", + "purple key", + "purple ball", + "purple door", + ] + + for sub_str in list_exclude_combinaison: + if sub_str in self.instrs.surface(self): + return True + return False + + def _regen_grid(self): + # Create the grid + self.grid.grid = [None] * self.width * self.height + + # For each row of rooms + for j in range(0, self.num_rows): + row = [] + + # For each column of rooms + for i in range(0, self.num_cols): + room = self.get_room(i, j) + # suppress doors and objects + room.doors = [None] * 4 + room.door_pos = [None] * 4 + room.neighbors = [None] * 4 + room.locked = False + room.objs = [] + row.append(room) + + # Generate the walls for this room + self.grid.wall_rect(*room.top, *room.size) + + self.room_grid.append(row) + + # For each row of rooms + for j in range(0, self.num_rows): + # For each column of rooms + for i in range(0, self.num_cols): + room = self.room_grid[j][i] + + x_l, y_l = (room.top[0] + 1, room.top[1] + 1) + x_m, y_m = ( + room.top[0] + room.size[0] - 1, + room.top[1] + room.size[1] - 1, + ) + + # Door positions, order is right, down, left, up + if i < self.num_cols - 1: + room.neighbors[0] = self.room_grid[j][i + 1] + room.door_pos[0] = (x_m, self._rand_int(y_l, y_m)) + if j < self.num_rows - 1: + room.neighbors[1] = self.room_grid[j + 1][i] + room.door_pos[1] = (self._rand_int(x_l, x_m), y_m) + if i > 0: + room.neighbors[2] = self.room_grid[j][i - 1] + room.door_pos[2] = room.neighbors[2].door_pos[0] + if j > 0: + room.neighbors[3] = self.room_grid[j - 1][i] + room.door_pos[3] = room.neighbors[3].door_pos[1] + + # The agent starts in the middle, facing right + self.agent_pos = ( + (self.num_cols // 2) * (self.room_size - 1) + (self.room_size // 2), + (self.num_rows // 2) * (self.room_size - 1) + (self.room_size // 2), + ) + self.agent_dir = 0 + + +class Level_MixedTrainLocalFrench(LevelGen): + """ + Same as MixedTrainLocal but in French + """ + + # TODO pas encore fini + + def __init__( + self, + room_size=8, + num_rows=1, + num_cols=1, + num_dists=8, + language="french", + instr_kinds=["action", "seq1"], + locations=False, + unblocking=False, + implicit_unlock=False, + **kwargs, + ): + action = self._rand_elem( + ["goto", "pickup", "open", "putnext", "pick up seq go to"] + ) + if action == "open": + num_cols = 2 + num_rows = 1 + # We add many distractors to increase the probability + # of ambiguous locations within the same room + super().__init__( + room_size=room_size, + num_rows=num_rows, + num_cols=num_cols, + num_dists=num_dists, + action_kinds=[action], + language=language, + instr_kinds=instr_kinds, + locations=locations, + unblocking=unblocking, + implicit_unlock=implicit_unlock, + **kwargs, + ) + + # ['goto', 'pickup', 'open', 'putnext', 'pick up seq go to'], + def gen_mission(self): + action = self._rand_elem(self.action_kinds) + mission_accepted = False + all_objects_reachable = False + if action == "open": + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + color_door = self._rand_elem( + ["jaune", "verte", "bleue", "violette"] + ) # red and grey excluded + self.add_locked_room(color_door) + self.connect_all() + + for j in range(self.num_rows): + for i in range(self.num_cols): + if self.get_room(i, j) is not self.locked_room: + self.add_distractors( + i, j, num_distractors=self.num_dists, all_unique=False + ) + + # The agent must be placed after all the object to respect constraints + while True: + self.place_agent() + start_room = self.room_from_pos(*self.agent_pos) + # Ensure that we are not placing the agent in the locked room + if start_room is self.locked_room: + continue + break + + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + + color_in_instr = self._rand_elem([None, color_door]) + + desc = ObjDesc("door", color_in_instr) + self.instrs = OpenInstr(desc) + + mission_accepted = not (self.exclude_substrings()) + + """if color_in_instr is None and mission_accepted and all_objects_reachable: + print(color_door)""" + + elif action == "goto": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 1, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj = self._rand_elem(objs) + self.instrs = GoToInstr(ObjDesc(obj.type, obj.color)) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "pickup": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 1, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj = self._rand_elem(objs) + while str(obj.type) == "door": + obj = self._rand_elem(objs) + self.instrs = PickupInstr(ObjDesc(obj.type, obj.color)) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "putnext": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 2, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj_1 = self._rand_elem(objs) + while str(obj_1.type) == "door": + obj_1 = self._rand_elem(objs) + desc1 = ObjDesc(obj_1.type, obj_1.color) + obj_2 = self._rand_elem(objs) + if obj_1.type == obj_2.type and obj_1.color == obj_2.color: + obj1s, poss = desc1.find_matching_objs(self) + if len(obj1s) < 2: + # if obj_1 is the only object with this description obj_2 has to be different + while obj_1.type == obj_2.type and obj_1.color == obj_2.color: + obj_2 = self._rand_elem(objs) + desc2 = ObjDesc(obj_2.type, obj_2.color) + self.instrs = PutNextInstr(desc1, desc2) + + mission_accepted = not (self.exclude_substrings()) + + elif action == "pick up seq go to": + self.num_cols = 1 + self.num_rows = 1 + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 2, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj_a = self._rand_elem(objs) + while str(obj_a.type) == "door": + obj_a = self._rand_elem(objs) + instr_a = PickupInstr(ObjDesc(obj_a.type, obj_a.color)) + obj_b = self._rand_elem(objs) + if obj_a.type == obj_b.type and obj_a.color == obj_b.color: + desc = ObjDesc(obj_a.type, obj_a.color) + objas, poss = desc.find_matching_objs(self) + if len(objas) < 2: + # if obj_a is the only object with this description obj_b has to be different + while obj_a.type == obj_b.type and obj_a.color == obj_b.color: + obj_b = self._rand_elem(objs) + instr_b = GoToInstr(ObjDesc(obj_b.type, obj_b.color)) + + type_instr = self._rand_elem(["Before", "After"]) + + if type_instr == "Before": + self.instrs = BeforeInstr(instr_a, instr_b) + else: + self.instrs = AfterInstr(instr_b, instr_a) + + mission_accepted = not (self.exclude_substrings()) + + def exclude_substrings(self): + # True if contains excluded substring + list_exclude_combinaison = [ + "boîte jaune", + "clef rouge", + "porte rouge", + "balle verte", + "porte grise", + ] + + for sub_str in list_exclude_combinaison: + str = self.instrs.surface(self) + if sub_str in self.instrs.surface(self): + return True + return False + + def _regen_grid(self): + # Create the grid + self.grid.grid = [None] * self.width * self.height + + # For each row of rooms + for j in range(0, self.num_rows): + row = [] + + # For each column of rooms + for i in range(0, self.num_cols): + room = self.get_room(i, j) + # suppress doors and objects + room.doors = [None] * 4 + room.door_pos = [None] * 4 + room.neighbors = [None] * 4 + room.locked = False + room.objs = [] + row.append(room) + + # Generate the walls for this room + self.grid.wall_rect(*room.top, *room.size) + + self.room_grid.append(row) + + # For each row of rooms + for j in range(0, self.num_rows): + # For each column of rooms + for i in range(0, self.num_cols): + room = self.room_grid[j][i] + + x_l, y_l = (room.top[0] + 1, room.top[1] + 1) + x_m, y_m = ( + room.top[0] + room.size[0] - 1, + room.top[1] + room.size[1] - 1, + ) + + # Door positions, order is right, down, left, up + if i < self.num_cols - 1: + room.neighbors[0] = self.room_grid[j][i + 1] + room.door_pos[0] = (x_m, self._rand_int(y_l, y_m)) + if j < self.num_rows - 1: + room.neighbors[1] = self.room_grid[j + 1][i] + room.door_pos[1] = (self._rand_int(x_l, x_m), y_m) + if i > 0: + room.neighbors[2] = self.room_grid[j][i - 1] + room.door_pos[2] = room.neighbors[2].door_pos[0] + if j > 0: + room.neighbors[3] = self.room_grid[j - 1][i] + room.door_pos[3] = room.neighbors[3].door_pos[1] + + # The agent starts in the middle, facing right + self.agent_pos = ( + (self.num_cols // 2) * (self.room_size - 1) + (self.room_size // 2), + (self.num_rows // 2) * (self.room_size - 1) + (self.room_size // 2), + ) + self.agent_dir = 0 + + +class Level_PickUpSeqGoToLocal(LevelGen): + """ + In order to test generalisation we only give to the agent the instruction: + seq restricted to pick up A then/before go to B with A and B without the following adj-noun pairs: + - yellow box + - red door/key + - green ball + - grey door + (for memory issue our agent only used the past 3 observations) + + Competencies: Seq never seen in MixedTrainLocal + """ + + def __init__( + self, + room_size=8, + num_rows=1, + num_cols=1, + num_dists=8, + instr_kinds=["seq1"], + locations=False, + unblocking=False, + implicit_unlock=False, + **kwargs, + ): + action = "pick up seq pick up " + + # We add many distractors to increase the probability + # of ambiguous locations within the same room + super().__init__( + room_size=room_size, + num_rows=num_rows, + num_cols=num_cols, + num_dists=num_dists, + action_kinds=[action], + instr_kinds=instr_kinds, + locations=locations, + unblocking=unblocking, + implicit_unlock=implicit_unlock, + **kwargs, + ) + + def gen_mission(self): + mission_accepted = False + all_objects_reachable = False + + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 2, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj_a = self._rand_elem(objs) + while str(obj_a.type) == "door": + obj_a = self._rand_elem(objs) + instr_a = PickupInstr(ObjDesc(obj_a.type, obj_a.color)) + obj_b = self._rand_elem(objs) + if obj_a.type == obj_b.type and obj_a.color == obj_b.color: + desc = ObjDesc(obj_a.type, obj_a.color) + objas, poss = desc.find_matching_objs(self) + if len(objas) < 2: + # if obj_a is the only object with this description obj_b has to be different + while obj_a.type == obj_b.type and obj_a.color == obj_b.color: + obj_b = self._rand_elem(objs) + instr_b = GoToInstr(ObjDesc(obj_b.type, obj_b.color)) + + type_instr = self._rand_elem(["Before", "After"]) + + if type_instr == "Before": + self.instrs = BeforeInstr(instr_a, instr_b) + else: + self.instrs = AfterInstr(instr_b, instr_a) + + mission_accepted = not (self.exclude_substrings()) + + def exclude_substrings(self): + # True if contains excluded substring + list_exclude_combinaison = [ + "yellow box", + "red key", + "red door", + "green ball", + "grey door", + ] + + for sub_str in list_exclude_combinaison: + if sub_str in self.instrs.surface(self): + return True + return False + + def _regen_grid(self): + # Create the grid + self.grid.grid = [None] * self.width * self.height + + # For each row of rooms + for j in range(0, self.num_rows): + row = [] + + # For each column of rooms + for i in range(0, self.num_cols): + room = self.get_room(i, j) + # suppress doors and objects + room.doors = [None] * 4 + room.door_pos = [None] * 4 + room.neighbors = [None] * 4 + room.locked = False + room.objs = [] + row.append(room) + + # Generate the walls for this room + self.grid.wall_rect(*room.top, *room.size) + + self.room_grid.append(row) + + # For each row of rooms + for j in range(0, self.num_rows): + # For each column of rooms + for i in range(0, self.num_cols): + room = self.room_grid[j][i] + + x_l, y_l = (room.top[0] + 1, room.top[1] + 1) + x_m, y_m = ( + room.top[0] + room.size[0] - 1, + room.top[1] + room.size[1] - 1, + ) + + # Door positions, order is right, down, left, up + if i < self.num_cols - 1: + room.neighbors[0] = self.room_grid[j][i + 1] + room.door_pos[0] = (x_m, self._rand_int(y_l, y_m)) + if j < self.num_rows - 1: + room.neighbors[1] = self.room_grid[j + 1][i] + room.door_pos[1] = (self._rand_int(x_l, x_m), y_m) + if i > 0: + room.neighbors[2] = self.room_grid[j][i - 1] + room.door_pos[2] = room.neighbors[2].door_pos[0] + if j > 0: + room.neighbors[3] = self.room_grid[j - 1][i] + room.door_pos[3] = room.neighbors[3].door_pos[1] + + # The agent starts in the middle, facing right + self.agent_pos = ( + (self.num_cols // 2) * (self.room_size - 1) + (self.room_size // 2), + (self.num_rows // 2) * (self.room_size - 1) + (self.room_size // 2), + ) + self.agent_dir = 0 + + +class Level_PickUpThenGoToLocal(LevelGen): + """ + In order to test generalisation we only give to the agent the instruction: + seq restricted to pick up A then go to B with A and B without the following adj-noun pairs: + - yellow box + - red door/key + - green ball + - grey door + (for memory issue our agent only used the past 3 observations) + + Competencies: Seq never seen in MixedTrainLocal + """ + + def __init__( + self, + room_size=8, + num_rows=1, + num_cols=1, + num_dists=8, + instr_kinds=["seq1"], + locations=False, + unblocking=False, + implicit_unlock=False, + **kwargs, + ): + action = "pick up seq pick up " + + # We add many distractors to increase the probability + # of ambiguous locations within the same room + super().__init__( + room_size=room_size, + num_rows=num_rows, + num_cols=num_cols, + num_dists=num_dists, + action_kinds=[action], + instr_kinds=instr_kinds, + locations=locations, + unblocking=unblocking, + implicit_unlock=implicit_unlock, + **kwargs, + ) + + def gen_mission(self): + mission_accepted = False + all_objects_reachable = False + + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 2, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj_a = self._rand_elem(objs) + while str(obj_a.type) == "door": + obj_a = self._rand_elem(objs) + instr_a = PickupInstr(ObjDesc(obj_a.type, obj_a.color)) + obj_b = self._rand_elem(objs) + if obj_a.type == obj_b.type and obj_a.color == obj_b.color: + desc = ObjDesc(obj_a.type, obj_a.color) + objas, poss = desc.find_matching_objs(self) + if len(objas) < 2: + # if obj_a is the only object with this description obj_b has to be different + while obj_a.type == obj_b.type and obj_a.color == obj_b.color: + obj_b = self._rand_elem(objs) + instr_b = GoToInstr(ObjDesc(obj_b.type, obj_b.color)) + + self.instrs = BeforeInstr(instr_a, instr_b) + + mission_accepted = not (self.exclude_substrings()) + + def exclude_substrings(self): + # True if contains excluded substring + list_exclude_combinaison = [ + "yellow box", + "red key", + "red door", + "green ball", + "grey door", + ] + + for sub_str in list_exclude_combinaison: + if sub_str in self.instrs.surface(self): + return True + return False + + def _regen_grid(self): + # Create the grid + self.grid.grid = [None] * self.width * self.height + + # For each row of rooms + for j in range(0, self.num_rows): + row = [] + + # For each column of rooms + for i in range(0, self.num_cols): + room = self.get_room(i, j) + # suppress doors and objects + room.doors = [None] * 4 + room.door_pos = [None] * 4 + room.neighbors = [None] * 4 + room.locked = False + room.objs = [] + row.append(room) + + # Generate the walls for this room + self.grid.wall_rect(*room.top, *room.size) + + self.room_grid.append(row) + + # For each row of rooms + for j in range(0, self.num_rows): + # For each column of rooms + for i in range(0, self.num_cols): + room = self.room_grid[j][i] + + x_l, y_l = (room.top[0] + 1, room.top[1] + 1) + x_m, y_m = ( + room.top[0] + room.size[0] - 1, + room.top[1] + room.size[1] - 1, + ) + + # Door positions, order is right, down, left, up + if i < self.num_cols - 1: + room.neighbors[0] = self.room_grid[j][i + 1] + room.door_pos[0] = (x_m, self._rand_int(y_l, y_m)) + if j < self.num_rows - 1: + room.neighbors[1] = self.room_grid[j + 1][i] + room.door_pos[1] = (self._rand_int(x_l, x_m), y_m) + if i > 0: + room.neighbors[2] = self.room_grid[j][i - 1] + room.door_pos[2] = room.neighbors[2].door_pos[0] + if j > 0: + room.neighbors[3] = self.room_grid[j - 1][i] + room.door_pos[3] = room.neighbors[3].door_pos[1] + + # The agent starts in the middle, facing right + self.agent_pos = ( + (self.num_cols // 2) * (self.room_size - 1) + (self.room_size // 2), + (self.num_rows // 2) * (self.room_size - 1) + (self.room_size // 2), + ) + self.agent_dir = 0 + + +class Level_GoToAfterPickUpLocal(LevelGen): + """ + In order to test generalisation we only give to the agent the instruction: + seq restricted to go to B after pickup A with A and B without the following adj-noun pairs: + - yellow box + - red door/key + - green ball + - grey door + (for memory issue our agent only used the past 3 observations) + + Competencies: Seq never seen in MixedTrainLocal + """ + + def __init__( + self, + room_size=8, + num_rows=1, + num_cols=1, + num_dists=8, + instr_kinds=["seq1"], + locations=False, + unblocking=False, + implicit_unlock=False, + **kwargs, + ): + action = "pick up seq pick up " + + # We add many distractors to increase the probability + # of ambiguous locations within the same room + super().__init__( + room_size=room_size, + num_rows=num_rows, + num_cols=num_cols, + num_dists=num_dists, + action_kinds=[action], + instr_kinds=instr_kinds, + locations=locations, + unblocking=unblocking, + implicit_unlock=implicit_unlock, + **kwargs, + ) + + def gen_mission(self): + mission_accepted = False + all_objects_reachable = False + + while not mission_accepted or not all_objects_reachable: + self._regen_grid() + self.place_agent() + objs = self.add_distractors( + num_distractors=self.num_dists + 2, all_unique=False + ) + all_objects_reachable = self.check_objs_reachable(raise_exc=False) + obj_a = self._rand_elem(objs) + while str(obj_a.type) == "door": + obj_a = self._rand_elem(objs) + instr_a = PickupInstr(ObjDesc(obj_a.type, obj_a.color)) + obj_b = self._rand_elem(objs) + if obj_a.type == obj_b.type and obj_a.color == obj_b.color: + desc = ObjDesc(obj_a.type, obj_a.color) + objas, poss = desc.find_matching_objs(self) + if len(objas) < 2: + # if obj_a is the only object with this description obj_b has to be different + while obj_a.type == obj_b.type and obj_a.color == obj_b.color: + obj_b = self._rand_elem(objs) + instr_b = GoToInstr(ObjDesc(obj_b.type, obj_b.color)) + + self.instrs = AfterInstr(instr_b, instr_a) + + mission_accepted = not (self.exclude_substrings()) + + def exclude_substrings(self): + # True if contains excluded substring + list_exclude_combinaison = [ + "yellow box", + "red key", + "red door", + "green ball", + "grey door", + ] + + for sub_str in list_exclude_combinaison: + if sub_str in self.instrs.surface(self): + return True + return False + + def _regen_grid(self): + # Create the grid + self.grid.grid = [None] * self.width * self.height + + # For each row of rooms + for j in range(0, self.num_rows): + row = [] + + # For each column of rooms + for i in range(0, self.num_cols): + room = self.get_room(i, j) + # suppress doors and objects + room.doors = [None] * 4 + room.door_pos = [None] * 4 + room.neighbors = [None] * 4 + room.locked = False + room.objs = [] + row.append(room) + + # Generate the walls for this room + self.grid.wall_rect(*room.top, *room.size) + + self.room_grid.append(row) + + # For each row of rooms + for j in range(0, self.num_rows): + # For each column of rooms + for i in range(0, self.num_cols): + room = self.room_grid[j][i] + + x_l, y_l = (room.top[0] + 1, room.top[1] + 1) + x_m, y_m = ( + room.top[0] + room.size[0] - 1, + room.top[1] + room.size[1] - 1, + ) + + # Door positions, order is right, down, left, up + if i < self.num_cols - 1: + room.neighbors[0] = self.room_grid[j][i + 1] + room.door_pos[0] = (x_m, self._rand_int(y_l, y_m)) + if j < self.num_rows - 1: + room.neighbors[1] = self.room_grid[j + 1][i] + room.door_pos[1] = (self._rand_int(x_l, x_m), y_m) + if i > 0: + room.neighbors[2] = self.room_grid[j][i - 1] + room.door_pos[2] = room.neighbors[2].door_pos[0] + if j > 0: + room.neighbors[3] = self.room_grid[j - 1][i] + room.door_pos[3] = room.neighbors[3].door_pos[1] + + # The agent starts in the middle, facing right + self.agent_pos = ( + (self.num_cols // 2) * (self.room_size - 1) + (self.room_size // 2), + (self.num_rows // 2) * (self.room_size - 1) + (self.room_size // 2), + ) + self.agent_dir = 0 diff --git a/minigrid/minigrid_env.py b/minigrid/minigrid_env.py index b50499b40..e4c0f1d3e 100755 --- a/minigrid/minigrid_env.py +++ b/minigrid/minigrid_env.py @@ -13,7 +13,13 @@ from gymnasium.core import ActType, ObsType from minigrid.core.actions import Actions -from minigrid.core.constants import COLOR_NAMES, DIR_TO_VEC, TILE_PIXELS +from minigrid.core.constants import ( + COLOR_NAMES, + COLOR_TO_IDX, + DIR_TO_VEC, + OBJECT_TO_IDX, + TILE_PIXELS, +) from minigrid.core.grid import Grid from minigrid.core.mission import MissionSpace from minigrid.core.world_object import Point, WorldObj @@ -45,6 +51,7 @@ def __init__( highlight: bool = True, tile_size: int = TILE_PIXELS, agent_pov: bool = False, + language="english", ): # Initialize mission self.mission = mission_space.sample() @@ -102,6 +109,9 @@ def __init__( self.see_through_walls = see_through_walls + # Language of the descriptions + self.language = language + # Current position and direction of the agent self.agent_pos: np.ndarray | tuple[int, int] = None self.agent_dir: int = None @@ -154,7 +164,10 @@ def reset( # Return first observation obs = self.gen_obs() - return obs, {} + # add info Episodic Knowledge to minigrid + info = self.gen_graph(move_forward=None) + + return obs, info def hash(self, size=16): """Compute a hash that uniquely identifies the current state of the environment. @@ -592,7 +605,15 @@ def step( obs = self.gen_obs() - return obs, reward, terminated, truncated, {} + # add info Episodic Knowledge to minigrid + move_forward = None + if action == self.actions.forward: + move_forward = False + if np.all(self.agent_pos == fwd_pos): + move_forward = True + info = self.gen_graph(move_forward=move_forward) + + return obs, reward, terminated, truncated, info def gen_obs_grid(self, agent_view_size=None): """ @@ -787,3 +808,227 @@ def render(self): def close(self): if self.window: pygame.quit() + + def gen_graph(self, move_forward=None): + grid, vis_mask = self.gen_obs_grid() + + # Encode the partially observable view into a numpy array + image = grid.encode(vis_mask) + # (OBJECT_TO_IDX[self.type], COLOR_TO_IDX[self.color], state) + # State, 0: open, 1: closed, 2: locked + if self.language == "english": + IDX_TO_STATE = {0: "open", 1: "closed", 2: "locked"} + IDX_TO_COLOR = dict(zip(COLOR_TO_IDX.values(), COLOR_TO_IDX.keys())) + IDX_TO_OBJECT = dict(zip(OBJECT_TO_IDX.values(), OBJECT_TO_IDX.keys())) + + elif self.language == "french": + IDX_TO_STATE = {0: "ouverte", 1: "fermée", 2: "fermée à clef"} + IDX_TO_COLOR = { + 0: "rouge", + 1: "verte", + 2: "bleue", + 3: "violette", + 4: "jaune", + 5: "grise", + } + IDX_TO_OBJECT = { + 0: "non visible", + 1: "vide", + 2: "mur", + 3: "sol", + 4: "porte", + 5: "clef", + 6: "balle", + 7: "boîte", + 8: "but", + 9: "lave", + 10: "agent", + } + + list_textual_descriptions = [] + + if self.carrying is not None: + # print('carrying') + if self.language == "english": + list_textual_descriptions.append( + f"You carry a {self.carrying.color} {self.carrying.type}" + ) + elif self.language == "french": + list_textual_descriptions.append( + "Tu portes une {} {}".format( + self.carrying.type, self.carrying.color + ) + ) + + # print('A agent position i: {}, j: {}'.format(self.agent_pos[0], self.agent_pos[1])) + agent_pos_vx, agent_pos_vy = self.get_view_coords( + self.agent_pos[0], self.agent_pos[1] + ) + # print('B agent position i: {}, j: {}'.format(agent_pos_vx, agent_pos_vy)) + + view_field_dictionary = dict() + + for i in range(image.shape[0]): + for j in range(image.shape[1]): + if image[i][j][0] != 0 and image[i][j][0] != 1 and image[i][j][0] != 2: + if i not in view_field_dictionary.keys(): + view_field_dictionary[i] = dict() + view_field_dictionary[i][j] = image[i][j] + else: + view_field_dictionary[i][j] = image[i][j] + + # Find the wall if any + # We describe a wall only if there is no objects between the agent and the wall in straight line + + # Find wall in front + j = agent_pos_vy - 1 + object_seen = False + while j >= 0 and not object_seen: + if image[agent_pos_vx][j][0] != 0 and image[agent_pos_vx][j][0] != 1: + if image[agent_pos_vx][j][0] == 2: + if self.language == "english": + list_textual_descriptions.append( + f"You see a wall {agent_pos_vy - j} step{'s' if agent_pos_vy - j > 1 else ''} forward" + ) + elif self.language == "french": + list_textual_descriptions.append( + f"Tu vois un mur à {agent_pos_vy - j} pas devant" + ) + object_seen = True + else: + object_seen = True + j -= 1 + # Find wall left + i = agent_pos_vx - 1 + object_seen = False + while i >= 0 and not object_seen: + if image[i][agent_pos_vy][0] != 0 and image[i][agent_pos_vy][0] != 1: + if image[i][agent_pos_vy][0] == 2: + if self.language == "english": + list_textual_descriptions.append( + f"You see a wall {agent_pos_vx - i} step{'s' if agent_pos_vx - i > 1 else ''} left" + ) + elif self.language == "french": + list_textual_descriptions.append( + f"Tu vois un mur à {agent_pos_vx - i} pas à gauche" + ) + object_seen = True + else: + object_seen = True + i -= 1 + # Find wall right + i = agent_pos_vx + 1 + object_seen = False + while i < image.shape[0] and not object_seen: + if image[i][agent_pos_vy][0] != 0 and image[i][agent_pos_vy][0] != 1: + if image[i][agent_pos_vy][0] == 2: + if self.language == "english": + list_textual_descriptions.append( + f"You see a wall {i - agent_pos_vx} step{'s' if i - agent_pos_vx > 1 else ''} right" + ) + elif self.language == "french": + list_textual_descriptions.append( + f"Tu vois un mur à {i - agent_pos_vx} pas à droite" + ) + object_seen = True + else: + object_seen = True + i += 1 + + # returns the position of seen objects relative to you + for i in view_field_dictionary.keys(): + for j in view_field_dictionary[i].keys(): + if i != agent_pos_vx or j != agent_pos_vy: + object = view_field_dictionary[i][j] + relative_position = dict() + + if i - agent_pos_vx > 0: + if self.language == "english": + relative_position["x_axis"] = ("right", i - agent_pos_vx) + elif self.language == "french": + relative_position["x_axis"] = ("à droite", i - agent_pos_vx) + elif i - agent_pos_vx == 0: + if self.language == "english": + relative_position["x_axis"] = ("face", 0) + elif self.language == "french": + relative_position["x_axis"] = ("en face", 0) + else: + if self.language == "english": + relative_position["x_axis"] = ("left", agent_pos_vx - i) + elif self.language == "french": + relative_position["x_axis"] = ("à gauche", agent_pos_vx - i) + if agent_pos_vy - j > 0: + if self.language == "english": + relative_position["y_axis"] = ("forward", agent_pos_vy - j) + elif self.language == "french": + relative_position["y_axis"] = ("devant", agent_pos_vy - j) + elif agent_pos_vy - j == 0: + if self.language == "english": + relative_position["y_axis"] = ("forward", 0) + elif self.language == "french": + relative_position["y_axis"] = ("devant", 0) + + distances = [] + if relative_position["x_axis"][0] in ["face", "en face"]: + distances.append( + ( + relative_position["y_axis"][1], + relative_position["y_axis"][0], + ) + ) + elif relative_position["y_axis"][1] == 0: + distances.append( + ( + relative_position["x_axis"][1], + relative_position["x_axis"][0], + ) + ) + else: + distances.append( + ( + relative_position["x_axis"][1], + relative_position["x_axis"][0], + ) + ) + distances.append( + ( + relative_position["y_axis"][1], + relative_position["y_axis"][0], + ) + ) + + description = "" + if object[0] != 4: # if it is not a door + if self.language == "english": + description = f"You see a {IDX_TO_COLOR[object[1]]} {IDX_TO_OBJECT[object[0]]} " + elif self.language == "french": + description = f"Tu vois une {IDX_TO_OBJECT[object[0]]} {IDX_TO_COLOR[object[1]]} " + + else: + if IDX_TO_STATE[object[2]] != 0: # if it is not open + if self.language == "english": + description = f"You see a {IDX_TO_STATE[object[2]]} {IDX_TO_COLOR[object[1]]} {IDX_TO_OBJECT[object[0]]} " + elif self.language == "french": + description = f"Tu vois une {IDX_TO_OBJECT[object[0]]} {IDX_TO_COLOR[object[1]]} {IDX_TO_STATE[object[2]]} " + + else: + if self.language == "english": + description = f"You see an {IDX_TO_STATE[object[2]]} {IDX_TO_COLOR[object[1]]} {IDX_TO_OBJECT[object[0]]} " + elif self.language == "french": + description = f"Tu vois une {IDX_TO_OBJECT[object[0]]} {IDX_TO_COLOR[object[1]]} {IDX_TO_STATE[object[2]]} " + + for _i, _distance in enumerate(distances): + if _i > 0: + if self.language == "english": + description += " and " + elif self.language == "french": + description += " et " + + if self.language == "english": + description += f"{_distance[0]} step{'s' if _distance[0] > 1 else ''} {_distance[1]}" + elif self.language == "french": + description += f"{_distance[0]} pas {_distance[1]}" + + list_textual_descriptions.append(description) + + return {"descriptions": list_textual_descriptions}