From 5ffe5fd9b3fd5a89bd2b534108686199129d04bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=98=95=E5=BD=A7?= <756309186@qq.com> Date: Fri, 23 Aug 2024 14:31:21 +0800 Subject: [PATCH 1/7] Copying project files directly, rather than copying them in an import-export fashion, can make copying projects many times faster --- gns3server/controller/project.py | 73 ++++++++++++++++++++++++++++++++ scripts/copy_tree.py | 15 +++++++ 2 files changed, 88 insertions(+) create mode 100644 scripts/copy_tree.py diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 644d9ba33..7c76f632d 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -1052,6 +1052,16 @@ async def duplicate(self, name=None, location=None, reset_mac_addresses=True): self.dump() assert self._status != "closed" + + try: + proj = await self._duplicate_fast(name, location, reset_mac_addresses) + if proj: + if previous_status == "closed": + await self.close() + return proj + except Exception as e: + raise aiohttp.web.HTTPConflict(text="Cannot duplicate project: {}".format(str(e))) + try: begin = time.time() @@ -1237,3 +1247,66 @@ def __json__(self): def __repr__(self): return "".format(self._name, self._id) + + async def _duplicate_fast(self, name=None, location=None, reset_mac_addresses=True): + # remote replication is not supported + if not sys.platform.startswith("linux") and not sys.platform.startswith("win"): + return None + for compute in self.computes: + if compute.id != "local": + log.warning("Duplicate fast not support remote compute: '{}'".format(compute.id)) + return None + # work dir + p_work = pathlib.Path(location or self.path).parent.absolute() + t0 = time.time() + new_project_id = str(uuid.uuid4()) + new_project_path = p_work.joinpath(new_project_id) + # copy dir + scripts_path = os.path.join(pathlib.Path(__file__).resolve().parent.parent.parent, 'scripts') + process = await asyncio.create_subprocess_exec('python', os.path.join(scripts_path, 'copy_tree.py'), '--src', + self.path, '--dst', + new_project_path.as_posix()) + await process.wait() + log.info("[FAST] Copy project: {} to: '{}', cost={}s".format(self.path, new_project_path, time.time() - t0)) + topology = json.loads(new_project_path.joinpath('{}.gns3'.format(self.name)).read_bytes()) + project_name = name or topology["name"] + # If the project name is already used we generate a new one + project_name = self.controller.get_free_project_name(project_name) + topology["name"] = project_name + # To avoid unexpected behavior (project start without manual operations just after import) + topology["auto_start"] = False + topology["auto_open"] = False + topology["auto_close"] = False + # change node ID + node_old_to_new = {} + for node in topology["topology"]["nodes"]: + new_node_id = str(uuid.uuid4()) + if "node_id" in node: + node_old_to_new[node["node_id"]] = new_node_id + _move_node_file(new_project_path, node["node_id"], new_node_id) + node["node_id"] = new_node_id + if reset_mac_addresses: + if "properties" in node and node["node_type"] != "docker": + for prop, value in node["properties"].items(): + # reset the MAC address + if prop in ("mac_addr", "mac_address"): + node["properties"][prop] = None + # change link ID + for link in topology["topology"]["links"]: + link["link_id"] = str(uuid.uuid4()) + for node in link["nodes"]: + node["node_id"] = node_old_to_new[node["node_id"]] + # Generate new drawings id + for drawing in topology["topology"]["drawings"]: + drawing["drawing_id"] = str(uuid.uuid4()) + + # And we dump the updated.gns3 + dot_gns3_path = new_project_path.joinpath('{}.gns3'.format(project_name)) + topology["project_id"] = new_project_id + with open(dot_gns3_path, "w+") as f: + json.dump(topology, f, indent=4) + + os.remove(new_project_path.joinpath('{}.gns3'.format(self.name))) + project = await self.controller.load_project(dot_gns3_path, load=False) + log.info("[FAST] Project '{}' duplicated in {:.4f} seconds".format(project.name, time.time() - t0)) + return project \ No newline at end of file diff --git a/scripts/copy_tree.py b/scripts/copy_tree.py new file mode 100644 index 000000000..d8d9e8fa7 --- /dev/null +++ b/scripts/copy_tree.py @@ -0,0 +1,15 @@ +import argparse +import shutil + + +# 复制目录 +def copy_tree(src, dst): + shutil.copytree(src, dst) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='for test') + parser.add_argument('--src', type=str, help='', default='') + parser.add_argument('--dst', type=str, help='', default='') + args = parser.parse_args() + copy_tree(args.src, args.dst) From 2dbde5df22be97ac64cb264620c42fe7cd40445d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=98=95=E5=BD=A7?= <756309186@qq.com> Date: Wed, 25 Sep 2024 20:27:46 +0800 Subject: [PATCH 2/7] Copying project files directly, rather than copying them in an import-export fashion, can make copying projects many times faster --- gns3server/controller/project.py | 8 +++----- scripts/copy_tree.py | 15 --------------- 2 files changed, 3 insertions(+), 20 deletions(-) delete mode 100644 scripts/copy_tree.py diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 7c76f632d..78cc14e5b 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -42,9 +42,11 @@ from ..utils.asyncio.pool import Pool from ..utils.asyncio import locking from ..utils.asyncio import aiozipstream +from ..utils.asyncio import wait_run_in_executor from .export_project import export_project from .import_project import import_project + import logging log = logging.getLogger(__name__) @@ -1262,11 +1264,7 @@ async def _duplicate_fast(self, name=None, location=None, reset_mac_addresses=Tr new_project_id = str(uuid.uuid4()) new_project_path = p_work.joinpath(new_project_id) # copy dir - scripts_path = os.path.join(pathlib.Path(__file__).resolve().parent.parent.parent, 'scripts') - process = await asyncio.create_subprocess_exec('python', os.path.join(scripts_path, 'copy_tree.py'), '--src', - self.path, '--dst', - new_project_path.as_posix()) - await process.wait() + await wait_run_in_executor(shutil.copytree, self.path, new_project_path.as_posix()) log.info("[FAST] Copy project: {} to: '{}', cost={}s".format(self.path, new_project_path, time.time() - t0)) topology = json.loads(new_project_path.joinpath('{}.gns3'.format(self.name)).read_bytes()) project_name = name or topology["name"] diff --git a/scripts/copy_tree.py b/scripts/copy_tree.py deleted file mode 100644 index d8d9e8fa7..000000000 --- a/scripts/copy_tree.py +++ /dev/null @@ -1,15 +0,0 @@ -import argparse -import shutil - - -# 复制目录 -def copy_tree(src, dst): - shutil.copytree(src, dst) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='for test') - parser.add_argument('--src', type=str, help='', default='') - parser.add_argument('--dst', type=str, help='', default='') - args = parser.parse_args() - copy_tree(args.src, args.dst) From a02b57698aca34fdbedda9dfd33b69131afa2b50 Mon Sep 17 00:00:00 2001 From: grossmj Date: Wed, 25 Sep 2024 19:45:14 +0700 Subject: [PATCH 3/7] Add missing imports --- gns3server/controller/project.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 78cc14e5b..f530ecb21 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import sys import re import os import json @@ -27,6 +28,7 @@ import aiofiles import tempfile import zipfile +import pathlib from uuid import UUID, uuid4 From 3a896b696476418d21d6a0bd6398617744a3b58a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=98=95=E5=BD=A7?= <756309186@qq.com> Date: Thu, 26 Sep 2024 08:26:08 +0800 Subject: [PATCH 4/7] Duplicate faster - 2 --- gns3server/controller/project.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index f530ecb21..43c3b1304 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -46,8 +46,7 @@ from ..utils.asyncio import aiozipstream from ..utils.asyncio import wait_run_in_executor from .export_project import export_project -from .import_project import import_project - +from .import_project import import_project, _move_node_file import logging log = logging.getLogger(__name__) From 996dad2f5c447da1c964942cc5b9de46d9331a52 Mon Sep 17 00:00:00 2001 From: grossmj Date: Thu, 26 Sep 2024 18:41:23 +0700 Subject: [PATCH 5/7] Support to reset MAC addresses for Docker nodes and some adjustments for fast duplication. --- gns3server/controller/export_project.py | 2 +- gns3server/controller/project.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gns3server/controller/export_project.py b/gns3server/controller/export_project.py index 9db433818..ae6126ad0 100644 --- a/gns3server/controller/export_project.py +++ b/gns3server/controller/export_project.py @@ -200,7 +200,7 @@ async def _patch_project_file(project, path, zstream, include_images, keep_compu if not keep_compute_ids: node["compute_id"] = "local" # To make project portable all node by default run on local - if "properties" in node and node["node_type"] != "docker": + if "properties" in node: for prop, value in node["properties"].items(): # reset the MAC address diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 43c3b1304..44099d9e6 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -1057,11 +1057,13 @@ async def duplicate(self, name=None, location=None, reset_mac_addresses=True): assert self._status != "closed" try: - proj = await self._duplicate_fast(name, location, reset_mac_addresses) + proj = await self._fast_duplication(name, location, reset_mac_addresses) if proj: if previous_status == "closed": await self.close() return proj + else: + log.info("Fast duplication failed, fallback to normal duplication") except Exception as e: raise aiohttp.web.HTTPConflict(text="Cannot duplicate project: {}".format(str(e))) @@ -1251,13 +1253,11 @@ def __json__(self): def __repr__(self): return "".format(self._name, self._id) - async def _duplicate_fast(self, name=None, location=None, reset_mac_addresses=True): - # remote replication is not supported - if not sys.platform.startswith("linux") and not sys.platform.startswith("win"): - return None + async def _fast_duplication(self, name=None, location=None, reset_mac_addresses=True): + # remote replication is not supported with remote computes for compute in self.computes: if compute.id != "local": - log.warning("Duplicate fast not support remote compute: '{}'".format(compute.id)) + log.warning("Fast duplication is not support with remote compute: '{}'".format(compute.id)) return None # work dir p_work = pathlib.Path(location or self.path).parent.absolute() @@ -1266,7 +1266,7 @@ async def _duplicate_fast(self, name=None, location=None, reset_mac_addresses=Tr new_project_path = p_work.joinpath(new_project_id) # copy dir await wait_run_in_executor(shutil.copytree, self.path, new_project_path.as_posix()) - log.info("[FAST] Copy project: {} to: '{}', cost={}s".format(self.path, new_project_path, time.time() - t0)) + log.info("Project content copied from '{}' to '{}' in {}s".format(self.path, new_project_path, time.time() - t0)) topology = json.loads(new_project_path.joinpath('{}.gns3'.format(self.name)).read_bytes()) project_name = name or topology["name"] # If the project name is already used we generate a new one @@ -1285,7 +1285,7 @@ async def _duplicate_fast(self, name=None, location=None, reset_mac_addresses=Tr _move_node_file(new_project_path, node["node_id"], new_node_id) node["node_id"] = new_node_id if reset_mac_addresses: - if "properties" in node and node["node_type"] != "docker": + if "properties" in node: for prop, value in node["properties"].items(): # reset the MAC address if prop in ("mac_addr", "mac_address"): @@ -1307,5 +1307,5 @@ async def _duplicate_fast(self, name=None, location=None, reset_mac_addresses=Tr os.remove(new_project_path.joinpath('{}.gns3'.format(self.name))) project = await self.controller.load_project(dot_gns3_path, load=False) - log.info("[FAST] Project '{}' duplicated in {:.4f} seconds".format(project.name, time.time() - t0)) + log.info("Project '{}' fast duplicated in {:.4f} seconds".format(project.name, time.time() - t0)) return project \ No newline at end of file From f7996d5e985b2e08729d8b2b6df96d051628403a Mon Sep 17 00:00:00 2001 From: grossmj Date: Fri, 27 Sep 2024 20:05:06 +0700 Subject: [PATCH 6/7] Fix tests --- gns3server/controller/export_project.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gns3server/controller/export_project.py b/gns3server/controller/export_project.py index ae6126ad0..7f62be18c 100644 --- a/gns3server/controller/export_project.py +++ b/gns3server/controller/export_project.py @@ -207,6 +207,9 @@ async def _patch_project_file(project, path, zstream, include_images, keep_compu if reset_mac_addresses and prop in ("mac_addr", "mac_address"): node["properties"][prop] = None + if node["node_type"] == "docker": + continue + if node["node_type"] == "iou": if not prop == "path": continue From cafdb2522bcfd3f098e5d816b37690b93182b652 Mon Sep 17 00:00:00 2001 From: grossmj Date: Sun, 29 Sep 2024 19:42:06 +0700 Subject: [PATCH 7/7] Add / update docstrings --- gns3server/controller/project.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 44099d9e6..3c5c5229d 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -1040,14 +1040,16 @@ async def duplicate(self, name=None, location=None, reset_mac_addresses=True): """ Duplicate a project - It's the save as feature of the 1.X. It's implemented on top of the - export / import features. It will generate a gns3p and reimport it. - It's a little slower but we have only one implementation to maintain. + Implemented on top of the export / import features. It will generate a gns3p and reimport it. + + NEW: fast duplication is used if possible (when there are no remote computes). + If not, the project is exported and reimported as explained above. :param name: Name of the new project. A new one will be generated in case of conflicts :param location: Parent directory of the new project :param reset_mac_addresses: Reset MAC addresses for the new project """ + # If the project was not open we open it temporary previous_status = self._status if self._status == "closed": @@ -1254,10 +1256,20 @@ def __repr__(self): return "".format(self._name, self._id) async def _fast_duplication(self, name=None, location=None, reset_mac_addresses=True): + """ + Fast duplication of a project. + + Copy the project files directly rather than in an import-export fashion. + + :param name: Name of the new project. A new one will be generated in case of conflicts + :param location: Parent directory of the new project + :param reset_mac_addresses: Reset MAC addresses for the new project + """ + # remote replication is not supported with remote computes for compute in self.computes: if compute.id != "local": - log.warning("Fast duplication is not support with remote compute: '{}'".format(compute.id)) + log.warning("Fast duplication is not supported with remote compute: '{}'".format(compute.id)) return None # work dir p_work = pathlib.Path(location or self.path).parent.absolute() @@ -1308,4 +1320,4 @@ async def _fast_duplication(self, name=None, location=None, reset_mac_addresses= os.remove(new_project_path.joinpath('{}.gns3'.format(self.name))) project = await self.controller.load_project(dot_gns3_path, load=False) log.info("Project '{}' fast duplicated in {:.4f} seconds".format(project.name, time.time() - t0)) - return project \ No newline at end of file + return project