Skip to content

Commit

Permalink
(#4): Add support for qemu 6.2: Use blockdev-backup instead of drive-…
Browse files Browse the repository at this point in the history
…backup
  • Loading branch information
abbbi committed Jan 29, 2024
1 parent 6f09701 commit 35a54b4
Show file tree
Hide file tree
Showing 6 changed files with 268 additions and 96 deletions.
30 changes: 27 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,10 @@ project:
- [Filesystem Freeze](#filesystem-freeze)
- [Backup Offline virtual machines](#backup-offline-virtual-machines)
- [Restore](#restore)
- [Misc commands](#misc-commands)
- [Misc commands and options](#misc-commands-and-options)
- [Compressing backups](#compressing-backups)
- [List devices suitable for backup](#list-devices-suitable-for-backup)
- [Including raw devices](#including-raw-devices)
- [List existing bitmaps](#list-existing-bitmaps)
- [Cleanup bitmaps](#cleanup-bitmaps)
- [Speed limit](#speed-limit)
Expand Down Expand Up @@ -190,15 +192,37 @@ time is possible:
qmprebase rebase --dir /tmp/backup/ide0-hd0 --until INC-1480542701
```

Misc commands
-------------
Misc commands and options
--------------------------

### Compressing backups

The `--compress` option can be used to enable compression for target files
during the `blockdev-backup` operation. This can save quite some storage space on
the created target images, but may slow down the backup operation.

```
qmpbackup --socket /tmp/vm backup [..] --compress
```

### List devices suitable for backup

```
qmpbackup --socket /tmp/vm info --show blockdev
```

### Including raw devices

Attached raw devices (format: raw) do not support incremental backup. The
only way to create backups for these devices is to create a complete full
backup.

By default `qmpbackup` will ignore such devices, but you can use the
`--include-raw` option to create a backup for those devices too.

Of course, if you create an incremental backup for these devices, the
complete image will be backed up.

### List existing bitmaps

To query existing bitmaps information use:
Expand Down
92 changes: 84 additions & 8 deletions libqmpbackup/image.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,17 @@
the LICENSE file in the top-level directory.
"""
import os
import json
import logging
import subprocess
from time import time

log = logging.getLogger(__name__)


def get_info(filename):
"""Query original qemu image information, can be used to re-create
the image during rebase operation with the same options as the
the image during backup operation with the same options as the
original one."""
try:
return subprocess.check_output(
Expand All @@ -34,14 +36,88 @@ def save_info(backupdir, blockdev):
"""Save qcow image information"""
for dev in blockdev:
infofile = f"{backupdir}/{dev.node}.config"
info = get_info(dev.filename)
try:
info = get_info(dev.filename)
except RuntimeError as errmsg:
log.warning("Unable to get qemu image info: [%s]", errmsg)
continue
with open(infofile, "wb+") as info_file:
info_file.write(info)
log.info("Saved image info: [%s]", infofile)
with open(infofile, "wb+") as info_file:
info_file.write(info)
log.info("Saved image info: [%s]", infofile)
except IOError as errmsg:
raise RuntimeError(f"Unable to store qcow config: [{errmsg}]") from errmsg
except Exception as errmsg:
raise RuntimeError(errmsg) from errmsg


def _get_options_cmd(backupdir, dev):
"""Read options to apply for backup target image from
qcow image info json output"""
opt = []
with open(f"{backupdir}/{dev.node}.config", "rb") as config_file:
qcow_config = json.loads(config_file.read().decode())

try:
opt.append("-o")
opt.append(f"compat={qcow_config['format-specific']['data']['compat']}")
except KeyError as errmsg:
log.warning("Unable apply QCOW specific compat option: [%s]", errmsg)

try:
opt.append("-o")
opt.append(f"cluster_size={qcow_config['cluster-size']}")
except KeyError as errmsg:
log.warning("Unable apply QCOW specific cluster_size option: [%s]", errmsg)

try:
if qcow_config["format-specific"]["data"]["lazy-refcounts"]:
opt.append("-o")
opt.append("lazy_refcounts=on")
except KeyError as errmsg:
log.warning("Unable apply QCOW specific lazy_refcounts option: [%s]", errmsg)

return opt


def create(argv, backupdir, blockdev):
"""Create target image used by qmp blockdev-backup image to dump
data and returns a list of target images per-device, which will
be used as parameter for QMP drive-backup operation"""
opt = []
dev_target = {}
timestamp = int(time())
for dev in blockdev:
targetdir = f"{backupdir}/{dev.node}/"
os.makedirs(targetdir, exist_ok=True)
filename = (
f"{argv.level.upper()}-{timestamp}-{os.path.basename(dev.filename)}.partial"
)
target = f"{targetdir}/{filename}"
if dev.format != "raw":
opt = opt + _get_options_cmd(backupdir, dev)

cmd = [
"qemu-img",
"create",
"-f",
f"{dev.format}",
f"{target}",
"-o",
f"size={dev.virtual_size}",
]
if dev.format != "raw":
cmd = cmd + opt

try:
log.info(
"Create target backup image: [%s], virtual size: [%s]",
target,
dev.virtual_size,
)
log.debug(cmd)
subprocess.check_output(cmd)
dev_target[dev.node] = target
except subprocess.CalledProcessError as errmsg:
raise RuntimeError from errmsg

return dev_target


def rebase(directory, dry_run, until):
Expand Down
111 changes: 62 additions & 49 deletions libqmpbackup/qmpcommon.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"""
import os
import logging
from time import sleep, time
from time import sleep
from qemu.qmp import EventListener
from libqmpbackup import fs

Expand All @@ -36,33 +36,45 @@ def transaction_bitmap_clear(self, node, name, **kwargs):
"block-dirty-bitmap-clear", node=node, name=name, **kwargs
)

def transaction_add_blockdev(self, name, driver, filename):
"""Return transaction action object for blockdev-add"""
return self.transaction_action(
"blockdev-add",
driver=driver,
name=name,
file={"driver": "file", "filename": filename},
)

def transaction_blockdev_create(self, name, driver, filename, size):
"""Return transaction action object for blockdev-add"""
return self.transaction_action(
"blockdev-create",
job_id=name,
name=name,
options={"driver": driver, "file": filename, "size": size},
)

def transaction_bitmap_add(self, node, name, **kwargs):
"""Return transaction action object for bitmap add"""
return self.transaction_action(
"block-dirty-bitmap-add", node=node, name=name, **kwargs
)

def prepare_transaction(self, argv, devices, backupdir):
async def prepare_target_devices(self, devices, target_files):
"""Create the required target devices for blockev-backup
operation"""
self.log.info("Prepare backup target devices")
for device in devices:
target = target_files[device.node]
targetdev = f"qmpbackup-{device.node}"

await self.qmp.execute(
"blockdev-add",
arguments={
"driver": device.format,
"node-name": targetdev,
"file": {"driver": "file", "filename": target},
},
)

async def remove_target_devices(self, devices):
"""Cleanup named devices after executing blockdev-backup
operation"""
self.log.info("Cleanup added backup target devices")
for device in devices:
targetdev = f"qmpbackup-{device.node}"

await self.qmp.execute(
"blockdev-del",
arguments={
"node-name": targetdev,
},
)

def prepare_transaction(self, argv, devices):
"""Prepare transaction steps"""
prefix = argv.level.upper()
sync = "full"
if argv.level == "inc":
sync = "incremental"
Expand All @@ -71,72 +83,76 @@ def prepare_transaction(self, argv, devices, backupdir):
persistent = True
if argv.level == "copy":
self.log.info("Copy backup: no persistent bitmap will be created.")
bitmap_prefix = "qmpbackup-copy"
bitmap_prefix = f"qmpbackup-{argv.level}"
persistent = False

actions = []
files = []
for device in devices:
timestamp = int(time())
targetdir = f"{backupdir}/{device.node}/"
os.makedirs(targetdir, exist_ok=True)
filename = (
f"{prefix}-{timestamp}-{os.path.basename(device.filename)}.partial"
)
target = f"{targetdir}/{filename}"
files.append(target)
targetdev = f"qmpbackup-{device.node}"
bitmap = f"{bitmap_prefix}-{device.node}"
job_id = f"{device.node}"

if (
not device.has_bitmap
and device.format != "raw"
and argv.level in ("full", "copy")
or device.has_bitmap
and argv.level in ("copy")
):
self.log.info("Creating new bitmap: %s", bitmap)
self.log.info(
"Creating new bitmap: [%s] for device [%s]", bitmap, device.node
)
actions.append(
self.transaction_bitmap_add(
device.node, bitmap, persistent=persistent
)
)

if device.has_bitmap and argv.level in ("full"):
self.log.debug("Clearing existing bitmap")
if device.has_bitmap and argv.level in ("full") and device.format != "raw":
self.log.info("Clearing existing bitmap for device: [%s]", device.node)
actions.append(self.transaction_bitmap_clear(device.node, bitmap))

if argv.level in ("full", "copy"):
compress = argv.compress
if device.format == "raw" and compress:
compress = False
self.log.info("Disabling compression for raw device: [%s]", device.node)

if argv.level in ("full", "copy") or (
argv.level == "inc" and device.format == "raw"
):
actions.append(
self.transaction_action(
"drive-backup",
"blockdev-backup",
device=device.node,
target=target,
sync=sync,
target=targetdev,
sync="full",
job_id=job_id,
speed=argv.speed_limit,
compress=compress,
)
)
else:
actions.append(
self.transaction_action(
"drive-backup",
"blockdev-backup",
bitmap=bitmap,
device=device.node,
target=target,
target=targetdev,
sync=sync,
job_id=job_id,
speed=argv.speed_limit,
compress=argv.compress,
)
)

self.log.debug("Created transaction: %s", actions)

return actions, files
return actions

async def backup(self, argv, devices, backupdir, qga):
async def backup(self, argv, devices, qga):
"""Start backup transaction, while backup is active,
watch for block status"""
actions, files = self.prepare_transaction(argv, devices, backupdir)
actions = self.prepare_transaction(argv, devices)
listener = EventListener(
(
"BLOCK_JOB_COMPLETED",
Expand Down Expand Up @@ -167,12 +183,9 @@ async def backup(self, argv, devices, backupdir, qga):
self.progress(jobs, devices)
sleep(1)

return files

async def do_query_block(self):
"""Return list of attached block devices"""
devices = await self.qmp.execute("query-block")
return devices
return await self.qmp.execute("query-block")

async def remove_bitmaps(self, blockdev, prefix="qmpbackup"):
"""Remove existing bitmaps for block devices"""
Expand All @@ -187,10 +200,10 @@ async def remove_bitmaps(self, blockdev, prefix="qmpbackup"):
if prefix not in bitmap_name:
self.log.debug("Ignoring bitmap: %s", bitmap_name)
continue
self.log.info("Removing bitmap: %s", f"{prefix}-{dev.node}")
self.log.info("Removing bitmap: %s", bitmap_name)
await self.qmp.execute(
"block-dirty-bitmap-remove",
arguments={"node": dev.node, "name": f"{prefix}-{dev.node}"},
arguments={"node": dev.node, "name": bitmap_name},
)

def progress(self, jobs, devices):
Expand Down
Loading

0 comments on commit 35a54b4

Please sign in to comment.