diff --git a/README.md b/README.md index 700d1bb..49cd070 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,10 @@ project: - [Filesystem Freeze](#filesystem-freeze) - [Backup Offline virtual machines](#backup-offline-virtual-machines) - [Restore](#restore) -- [Misc commands](#misc-commands) +- [Misc commands and options](#misc-commands-and-options) + - [Compressing backups](#compressing-backups) - [List devices suitable for backup](#list-devices-suitable-for-backup) + - [Including raw devices](#including-raw-devices) - [List existing bitmaps](#list-existing-bitmaps) - [Cleanup bitmaps](#cleanup-bitmaps) - [Speed limit](#speed-limit) @@ -190,8 +192,18 @@ time is possible: qmprebase rebase --dir /tmp/backup/ide0-hd0 --until INC-1480542701 ``` -Misc commands -------------- +Misc commands and options +-------------------------- + +### Compressing backups + +The `--compress` option can be used to enable compression for target files +during the `blockdev-backup` operation. This can save quite some storage space on +the created target images, but may slow down the backup operation. + +``` + qmpbackup --socket /tmp/vm backup [..] --compress +``` ### List devices suitable for backup @@ -199,6 +211,18 @@ Misc commands qmpbackup --socket /tmp/vm info --show blockdev ``` +### Including raw devices + +Attached raw devices (format: raw) do not support incremental backup. The +only way to create backups for these devices is to create a complete full +backup. + +By default `qmpbackup` will ignore such devices, but you can use the +`--include-raw` option to create a backup for those devices too. + +Of course, if you create an incremental backup for these devices, the +complete image will be backed up. + ### List existing bitmaps To query existing bitmaps information use: diff --git a/libqmpbackup/image.py b/libqmpbackup/image.py index f67fece..a8dfa58 100644 --- a/libqmpbackup/image.py +++ b/libqmpbackup/image.py @@ -12,15 +12,17 @@ the LICENSE file in the top-level directory. """ import os +import json import logging import subprocess +from time import time log = logging.getLogger(__name__) def get_info(filename): """Query original qemu image information, can be used to re-create - the image during rebase operation with the same options as the + the image during backup operation with the same options as the original one.""" try: return subprocess.check_output( @@ -34,14 +36,88 @@ def save_info(backupdir, blockdev): """Save qcow image information""" for dev in blockdev: infofile = f"{backupdir}/{dev.node}.config" + info = get_info(dev.filename) try: - info = get_info(dev.filename) - except RuntimeError as errmsg: - log.warning("Unable to get qemu image info: [%s]", errmsg) - continue - with open(infofile, "wb+") as info_file: - info_file.write(info) - log.info("Saved image info: [%s]", infofile) + with open(infofile, "wb+") as info_file: + info_file.write(info) + log.info("Saved image info: [%s]", infofile) + except IOError as errmsg: + raise RuntimeError(f"Unable to store qcow config: [{errmsg}]") from errmsg + except Exception as errmsg: + raise RuntimeError(errmsg) from errmsg + + +def _get_options_cmd(backupdir, dev): + """Read options to apply for backup target image from + qcow image info json output""" + opt = [] + with open(f"{backupdir}/{dev.node}.config", "rb") as config_file: + qcow_config = json.loads(config_file.read().decode()) + + try: + opt.append("-o") + opt.append(f"compat={qcow_config['format-specific']['data']['compat']}") + except KeyError as errmsg: + log.warning("Unable apply QCOW specific compat option: [%s]", errmsg) + + try: + opt.append("-o") + opt.append(f"cluster_size={qcow_config['cluster-size']}") + except KeyError as errmsg: + log.warning("Unable apply QCOW specific cluster_size option: [%s]", errmsg) + + try: + if qcow_config["format-specific"]["data"]["lazy-refcounts"]: + opt.append("-o") + opt.append("lazy_refcounts=on") + except KeyError as errmsg: + log.warning("Unable apply QCOW specific lazy_refcounts option: [%s]", errmsg) + + return opt + + +def create(argv, backupdir, blockdev): + """Create target image used by qmp blockdev-backup image to dump + data and returns a list of target images per-device, which will + be used as parameter for QMP drive-backup operation""" + opt = [] + dev_target = {} + timestamp = int(time()) + for dev in blockdev: + targetdir = f"{backupdir}/{dev.node}/" + os.makedirs(targetdir, exist_ok=True) + filename = ( + f"{argv.level.upper()}-{timestamp}-{os.path.basename(dev.filename)}.partial" + ) + target = f"{targetdir}/{filename}" + if dev.format != "raw": + opt = opt + _get_options_cmd(backupdir, dev) + + cmd = [ + "qemu-img", + "create", + "-f", + f"{dev.format}", + f"{target}", + "-o", + f"size={dev.virtual_size}", + ] + if dev.format != "raw": + cmd = cmd + opt + + try: + log.info( + "Create target backup image: [%s], virtual size: [%s]", + target, + dev.virtual_size, + ) + log.debug(cmd) + subprocess.check_output(cmd) + dev_target[dev.node] = target + except subprocess.CalledProcessError as errmsg: + raise RuntimeError from errmsg + + return dev_target def rebase(directory, dry_run, until): diff --git a/libqmpbackup/qmpcommon.py b/libqmpbackup/qmpcommon.py index 9657ee7..4e0fa53 100644 --- a/libqmpbackup/qmpcommon.py +++ b/libqmpbackup/qmpcommon.py @@ -10,7 +10,7 @@ """ import os import logging -from time import sleep, time +from time import sleep from qemu.qmp import EventListener from libqmpbackup import fs @@ -36,33 +36,45 @@ def transaction_bitmap_clear(self, node, name, **kwargs): "block-dirty-bitmap-clear", node=node, name=name, **kwargs ) - def transaction_add_blockdev(self, name, driver, filename): - """Return transaction action object for blockdev-add""" - return self.transaction_action( - "blockdev-add", - driver=driver, - name=name, - file={"driver": "file", "filename": filename}, - ) - - def transaction_blockdev_create(self, name, driver, filename, size): - """Return transaction action object for blockdev-add""" - return self.transaction_action( - "blockdev-create", - job_id=name, - name=name, - options={"driver": driver, "file": filename, "size": size}, - ) - def transaction_bitmap_add(self, node, name, **kwargs): """Return transaction action object for bitmap add""" return self.transaction_action( "block-dirty-bitmap-add", node=node, name=name, **kwargs ) - def prepare_transaction(self, argv, devices, backupdir): + async def prepare_target_devices(self, devices, target_files): + """Create the required target devices for blockev-backup + operation""" + self.log.info("Prepare backup target devices") + for device in devices: + target = target_files[device.node] + targetdev = f"qmpbackup-{device.node}" + + await self.qmp.execute( + "blockdev-add", + arguments={ + "driver": device.format, + "node-name": targetdev, + "file": {"driver": "file", "filename": target}, + }, + ) + + async def remove_target_devices(self, devices): + """Cleanup named devices after executing blockdev-backup + operation""" + self.log.info("Cleanup added backup target devices") + for device in devices: + targetdev = f"qmpbackup-{device.node}" + + await self.qmp.execute( + "blockdev-del", + arguments={ + "node-name": targetdev, + }, + ) + + def prepare_transaction(self, argv, devices): """Prepare transaction steps""" - prefix = argv.level.upper() sync = "full" if argv.level == "inc": sync = "incremental" @@ -71,72 +83,76 @@ def prepare_transaction(self, argv, devices, backupdir): persistent = True if argv.level == "copy": self.log.info("Copy backup: no persistent bitmap will be created.") - bitmap_prefix = "qmpbackup-copy" + bitmap_prefix = f"qmpbackup-{argv.level}" persistent = False actions = [] - files = [] for device in devices: - timestamp = int(time()) - targetdir = f"{backupdir}/{device.node}/" - os.makedirs(targetdir, exist_ok=True) - filename = ( - f"{prefix}-{timestamp}-{os.path.basename(device.filename)}.partial" - ) - target = f"{targetdir}/{filename}" - files.append(target) + targetdev = f"qmpbackup-{device.node}" bitmap = f"{bitmap_prefix}-{device.node}" job_id = f"{device.node}" if ( not device.has_bitmap + and device.format != "raw" and argv.level in ("full", "copy") or device.has_bitmap and argv.level in ("copy") ): - self.log.info("Creating new bitmap: %s", bitmap) + self.log.info( + "Creating new bitmap: [%s] for device [%s]", bitmap, device.node + ) actions.append( self.transaction_bitmap_add( device.node, bitmap, persistent=persistent ) ) - if device.has_bitmap and argv.level in ("full"): - self.log.debug("Clearing existing bitmap") + if device.has_bitmap and argv.level in ("full") and device.format != "raw": + self.log.info("Clearing existing bitmap for device: [%s]", device.node) actions.append(self.transaction_bitmap_clear(device.node, bitmap)) - if argv.level in ("full", "copy"): + compress = argv.compress + if device.format == "raw" and compress: + compress = False + self.log.info("Disabling compression for raw device: [%s]", device.node) + + if argv.level in ("full", "copy") or ( + argv.level == "inc" and device.format == "raw" + ): actions.append( self.transaction_action( - "drive-backup", + "blockdev-backup", device=device.node, - target=target, - sync=sync, + target=targetdev, + sync="full", job_id=job_id, speed=argv.speed_limit, + compress=compress, ) ) else: actions.append( self.transaction_action( - "drive-backup", + "blockdev-backup", bitmap=bitmap, device=device.node, - target=target, + target=targetdev, sync=sync, job_id=job_id, speed=argv.speed_limit, + compress=argv.compress, ) ) self.log.debug("Created transaction: %s", actions) - return actions, files + return actions - async def backup(self, argv, devices, backupdir, qga): + async def backup(self, argv, devices, qga): """Start backup transaction, while backup is active, watch for block status""" - actions, files = self.prepare_transaction(argv, devices, backupdir) + actions = self.prepare_transaction(argv, devices) listener = EventListener( ( "BLOCK_JOB_COMPLETED", @@ -167,12 +183,9 @@ async def backup(self, argv, devices, backupdir, qga): self.progress(jobs, devices) sleep(1) - return files - async def do_query_block(self): """Return list of attached block devices""" - devices = await self.qmp.execute("query-block") - return devices + return await self.qmp.execute("query-block") async def remove_bitmaps(self, blockdev, prefix="qmpbackup"): """Remove existing bitmaps for block devices""" @@ -187,10 +200,10 @@ async def remove_bitmaps(self, blockdev, prefix="qmpbackup"): if prefix not in bitmap_name: self.log.debug("Ignoring bitmap: %s", bitmap_name) continue - self.log.info("Removing bitmap: %s", f"{prefix}-{dev.node}") + self.log.info("Removing bitmap: %s", bitmap_name) await self.qmp.execute( "block-dirty-bitmap-remove", - arguments={"node": dev.node, "name": f"{prefix}-{dev.node}"}, + arguments={"node": dev.node, "name": bitmap_name}, ) def progress(self, jobs, devices): diff --git a/libqmpbackup/vm.py b/libqmpbackup/vm.py index 1571262..66d9f42 100644 --- a/libqmpbackup/vm.py +++ b/libqmpbackup/vm.py @@ -12,27 +12,28 @@ """ import os import logging -from collections import namedtuple +from dataclasses import dataclass log = logging.getLogger(__name__) -def get_block_devices(blockinfo, excluded_disks, included_disks): +@dataclass +class BlockDev: + """Block device information""" + + node: str + format: str + filename: str + backing_image: str + has_bitmap: bool + bitmaps: list + virtual_size: int + + +def get_block_devices(blockinfo, argv, excluded_disks, included_disks): """Get a list of block devices that we can create a bitmap for, currently we only get inserted qcow based images """ - BlockDev = namedtuple( - "BlockDev", - [ - "node", - "format", - "filename", - "backing_image", - "has_bitmap", - "bitmaps", - "virtual_size", - ], - ) blockdevs = [] for device in blockinfo: bitmaps = None @@ -44,7 +45,7 @@ def get_block_devices(blockinfo, excluded_disks, included_disks): inserted = device["inserted"] base_filename = os.path.basename(inserted["image"]["filename"]) - if inserted["drv"] == "raw": + if inserted["drv"] == "raw" and not argv.include_raw: log.warning( "Excluding device with raw format from backup: [%s:%s]", device["device"], diff --git a/qmpbackup b/qmpbackup index 1cf9476..68fd8c8 100755 --- a/qmpbackup +++ b/qmpbackup @@ -17,6 +17,7 @@ import asyncio import signal import argparse from datetime import datetime +from dataclasses import asdict from qemu.qmp import protocol, QMPClient from libqmpbackup.qmpcommon import QmpCommon @@ -145,6 +146,20 @@ async def main(): help="speed limit in bytes / second", required=False, ) + parser_backup.add_argument( + "--compress", + action="store_true", + default=False, + help="Attempt to compress data if target image format supports it", + required=False, + ) + parser_backup.add_argument( + "--include-raw", + action="store_true", + default=False, + help="Include raw images in backup.", + required=False, + ) parser_cleanup = subparsers.add_parser("cleanup", help="cleanup functions") parser_cleanup.set_defaults(which="cleanup") parser_cleanup.add_argument( @@ -212,10 +227,19 @@ async def main(): log.debug("Excluded disks: %s", excluded_disks) if argv.include is not None: included_disks = argv.include.split(",") - log.debug("Backing up only specified disks: %s", included_disks) + log.debug("Saving only specified disks: %s", included_disks) + if argv.compress: + log.info("Enabling compress option for backup operation.") + if argv.include_raw: + log.info("Including raw devices in backup operation.") + + if action == "info": + argv.include_raw = True + if action == "cleanup": + argv.include_raw = False blockdev = vm.get_block_devices( - await qemu_client.do_query_block(), excluded_disks, included_disks + await qemu_client.do_query_block(), argv, excluded_disks, included_disks ) loop = asyncio.get_event_loop() @@ -235,7 +259,7 @@ async def main(): if argv.show == "blockdev": log.info("Attached block devices:") for dev in blockdev: - log.info("%s", lib.json_pp(dev._asdict())) + log.info("%s", lib.json_pp(asdict(dev))) if argv.show == "bitmaps": for dev in blockdev: if not dev.bitmaps: @@ -243,11 +267,10 @@ async def main(): continue log.info("%s:", dev.node) log.info("%s", lib.json_pp(dev.bitmaps)) - sys.exit(0) if action == "cleanup": + log.info("Removing all existent bitmaps.") await qemu_client.remove_bitmaps(blockdev) - log.info("Bitmaps for all devices have been removed") if action == "backup": if argv.quiesce and not argv.agentsocket: @@ -263,10 +286,6 @@ async def main(): log.info("New monthly directory will be created: %s", backupdir) new_monthly = True - if os.path.isfile(backupdir): - log.fatal("Backup target must be an directory.") - sys.exit(1) - try: os.makedirs(backupdir, exist_ok=True) except OSError as errmsg: @@ -274,22 +293,28 @@ async def main(): sys.exit(1) for device in blockdev: + tdir = f"{backupdir}/{device.node}" if device.backing_image is True: log.error( 'Active backing image for disk "%s", please commit any snapshots before starting a new chain.', device.node, ) sys.exit(1) - if device.has_bitmap is False and argv.level == "inc": + if ( + device.has_bitmap is False + and argv.level == "inc" + and device.format != "raw" + ): log.error( - "Incremental backup requested but no active bitmap has been found." + "[%s:%s] Incremental backup requested but no active bitmap has been found.", + dev.node, + dev.filename, ) sys.exit(1) - if argv.level == "auto": if ( device.has_bitmap - and not lib.has_full(f"{backupdir}/{device.node}") + and not lib.has_full(tdir) and new_monthly is False ): log.error( @@ -303,8 +328,12 @@ async def main(): argv.level = "inc" log.info("Auto backup mode set to: %s", argv.level) - if argv.level == "inc": + if not lib.has_full(tdir): + log.info( + "No full backup found in [%s]: Execute full backup first.", tdir + ) + sys.exit(1) if lib.check_for_partial(backupdir, device.node): log.error( "Partial backup found in [%s/%s], possible broken backup chain. Execute new full backup", @@ -315,7 +344,12 @@ async def main(): log.info("Backup target directory: %s", backupdir) - image.save_info(backupdir, blockdev) + try: + image.save_info(backupdir, blockdev) + target_files = image.create(argv, backupdir, blockdev) + except RuntimeError as errmsg: + log.fatal(errmsg) + sys.exit(1) qga = False if argv.agentsocket and argv.quiesce: @@ -324,16 +358,19 @@ async def main(): fs.quiesce(qga) try: - files = await qemu_client.backup(argv, blockdev, backupdir, qga) + await qemu_client.prepare_target_devices(blockdev, target_files) + await qemu_client.backup(argv, blockdev, qga) except Exception as errmsg: log.fatal("Error executing backup: %s", errmsg) + sys.exit(1) + finally: + await qemu_client.remove_target_devices(blockdev) if qga is not False: fs.thaw(qga) - sys.exit(1) if argv.level == "copy": blockdev = vm.get_block_devices( - await qemu_client.do_query_block(), excluded_disks, included_disks + await qemu_client.do_query_block(), argv, excluded_disks, included_disks ) log.info("Removing non-persistent bitmaps used for copy backup.") await qemu_client.remove_bitmaps(blockdev, prefix="qmpbackup-copy") @@ -345,7 +382,7 @@ async def main(): sys.exit(1) log.info("Renaming partial files") - for saveset in files: + for _, saveset in target_files.items(): new_filename = saveset.replace(".partial", "") try: os.rename(saveset, new_filename) diff --git a/t/runtest b/t/runtest index 4f00ef4..4ae54c2 100755 --- a/t/runtest +++ b/t/runtest @@ -83,8 +83,22 @@ rm -rf /tmp/backup rm -rf /tmp/copy_backup ../qmpbackup --agent-socket $AGENT_SOCKET --socket $QMP_SOCKET backup --level copy --target /tmp/copy_backup/ --quiesce -exist_files /tmp/backup//ide0-hd0/FULL* -exist_files /tmp/backup//ide0-hd1/FULL* +exist_files /tmp/backup//ide0-hd0/FULL* +exist_files /tmp/backup//ide0-hd1/FULL* + + +# compress option +rm -rf /tmp/compressed_backup/ +../qmpbackup --agent-socket $AGENT_SOCKET --socket $QMP_SOCKET backup --level copy --target /tmp/compressed_backup/ --quiesce --compress +rm -rf /tmp/compressed_backup/ + +# compress option +rm -rf /tmp/raw_backup/ +../qmpbackup --agent-socket $AGENT_SOCKET --socket $QMP_SOCKET backup --level full --target /tmp/raw_backup/ --quiesce --include-raw +exist_files /tmp/raw_backup//ide1-hd0/FULL* +rm -rf /tmp/raw_backup/ + + # create /tmp/incdata1 within the guest, execute further # incremental backups @@ -102,6 +116,12 @@ echo "------------------------------------------------" ../qmpbackup --agent-socket $AGENT_SOCKET --socket $QMP_SOCKET backup --level auto --target /tmp/backup/ --quiesce +echo "------------------------------------------------" +python3 agent.py incdata3 +echo "------------------------------------------------" +../qmpbackup --agent-socket $AGENT_SOCKET --socket $QMP_SOCKET backup --level auto --target /tmp/backup/ --quiesce --compress + + rm -rf /tmp/monthly ../qmpbackup --socket $QMP_SOCKET backup --level auto --monthly --target /tmp/monthly/ exist_files /tmp/monthly/*/*/FULL* @@ -164,5 +184,6 @@ rm -f /tmp/diff virt-diff -a "$ORIGINAL_FILE" -A "$RESTORED_FILE" > /tmp/diff grep -m 1 incdata1 /tmp/diff grep -m 1 incdata2 /tmp/diff +grep -m 1 incdata3 /tmp/diff echo "OK"