From b156b1fe14ac819a101129bd8c2fc8e67c332328 Mon Sep 17 00:00:00 2001 From: Sean Eric Fagan Date: Fri, 9 Jun 2017 19:35:16 -0700 Subject: [PATCH 1/6] Add support for glacier transitions. Also try to create the specified bucket if it does not already exist. --- z3/snap.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 6 deletions(-) diff --git a/z3/snap.py b/z3/snap.py index 5f01bae..4a2576c 100644 --- a/z3/snap.py +++ b/z3/snap.py @@ -62,13 +62,13 @@ class S3Snapshot(object): MISSING_PARENT = 'missing parent' PARENT_BROKEN = 'parent broken' - def __init__(self, name, metadata, manager, size): + def __init__(self, name, metadata, manager, size, key=None): self.name = name self._metadata = metadata self._mgr = manager self._reason_broken = None self.size = size - + self.key = key def __repr__(self): if self.is_full: return "".format(self.name) @@ -140,7 +140,7 @@ def _snapshots(self): for key in self.bucket.list(prefix): key = self.bucket.get_key(key.key) name = key.key[strip_chars:] - snapshots[name] = S3Snapshot(name, metadata=key.metadata, manager=self, size=key.size) + snapshots[name] = S3Snapshot(name, metadata=key.metadata, manager=self, size=key.size, key=key) return snapshots def list(self): @@ -405,6 +405,11 @@ def restore(self, snap_name, dry_run=False, force=False): current_snap = self.s3_manager.get(snap_name) if current_snap is None: raise Exception('no such snapshot "{}"'.format(snap_name)) + if current_snap.key.ongoing_restore == True: + raise Exception('snapshot {} is currently being restore from glocier; try again later'.format(snap_name)) + if current_snap.key.storage_class == "GLACIER": + current_snap.key.restore(days=5) + raise Exception('snapshot {} is currently in glacier storage, requesting transfer now'.format(snap_name)) to_restore = [] while True: z_snap = self.zfs_manager.get(current_snap.name) @@ -519,10 +524,16 @@ def restore(bucket, s3_prefix, filesystem, snapshot_prefix, snapshot, dry, force def parse_args(): + def to_bool(s): + if s.lower() in ("yes", "1", "true", "t", "y"): + return True + return False + cfg = get_config() parser = argparse.ArgumentParser( description='list z3 snapshots', ) + parser.register('type', 'bool', to_bool) parser.add_argument('--s3-prefix', dest='s3_prefix', default=cfg.get('S3_PREFIX', 'z3-backup/'), @@ -536,6 +547,12 @@ def parse_args(): default=None, help=('Only operate on snapshots that start with this prefix. ' 'Defaults to zfs-auto-snap:daily.')) + parser.add_argument("--use-glacier", + dest='use_glacier', + type=bool, + default=to_bool(cfg.get("USE_GLACIER", "False")), + help='Use glacier for storage') + subparsers = parser.add_subparsers(help='sub-command help', dest='subcommand') backup_parser = subparsers.add_parser( @@ -574,7 +591,7 @@ def main(): args = parse_args() try: - s3_key_id, s3_secret, bucket = cfg['S3_KEY_ID'], cfg['S3_SECRET'], cfg['BUCKET'] + s3_key_id, s3_secret, bucket_name = cfg['S3_KEY_ID'], cfg['S3_SECRET'], cfg['BUCKET'] extra_config = {} if 'HOST' in cfg: @@ -583,8 +600,60 @@ def main(): sys.stderr.write("Configuration error! {} is not set.\n".format(err)) sys.exit(1) - bucket = boto.connect_s3(s3_key_id, s3_secret, **extra_config).get_bucket(bucket) - + s3 = boto.connect_s3(s3_key_id, s3_secret, **extra_config) + try: + bucket = s3.get_bucket(bucket_name) + except boto.exception.S3ResponseError as e: + if e.error_code == 'NoSuchBucket': + # Let's try creating it + bucket = s3.create_bucket(bucket_name) + print("Created bucket {}: {}".format(bucket_name, bucket), file=sys.stderr) + else: + raise + + try: + lifecycle = bucket.get_lifecycle_config() + except boto.exception.S3ResponseError: + lifecycle = None + + # See if we have a lifecycle rule for glacier. + # The rule name will depend on the S3_PREFIX -- if + # that's empty, then the rule is just "z3-transition"; + # otherise, it is "z3 transition ${S3_PREFIX}" (but with + # '/' converted to ' '). + lifecycle_rule_name = "z3 transition" + if args.s3_prefix: + lifecycle_rule_name += " " + args.s3_prefix.replace("/", " ") + while lifecycle_rule_name.endswith(" "): + lifecycle_rule_name = lifecycle_rule_name[:-1] + + rule_index = None + for indx, rule in enumerate(lifecycle or []): + if rule.id == lifecycle_rule_name: + rule_index = indx + break + + if not args.use_glacier: + # If we don't use glacier, we want to remove the lifecycle policy + # if it exists + if rule_index is not None: + lifecycle.pop(rule_index) + else: + if rule_index is None: + # Okay, we need to add a lifecycle + if lifecycle is None: + lifecycle = boto.s3.lifecycle.Lifecycle() + transition=boto.s3.lifecycle.Transition(days=1, storage_class="GLACIER") + print("trasition rule = {}".format(transition), file=sys.stderr) + print("prefix = {}".format(args.s3_prefix or None), file=sys.stderr) + lifecycle.add_rule(id=lifecycle_rule_name, + status="Enabled", + prefix=args.s3_prefix or None, + transition=transition) + print("lifecycle = {}".format(lifecycle.to_xml()), file=sys.stderr) + if lifecycle is not None: + bucket.configure_lifecycle(lifecycle) + fs_section = "fs:{}".format(args.filesystem) if args.snapshot_prefix is None: snapshot_prefix = cfg.get("SNAPSHOT_PREFIX", section=fs_section) From 4c80351fafc824c855798169293995a5f3d0a0e1 Mon Sep 17 00:00:00 2001 From: Sean Eric Fagan Date: Fri, 9 Jun 2017 19:50:18 -0700 Subject: [PATCH 2/6] Fix a comment. --- z3/snap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/z3/snap.py b/z3/snap.py index 4a2576c..b3570ed 100644 --- a/z3/snap.py +++ b/z3/snap.py @@ -618,7 +618,7 @@ def main(): # See if we have a lifecycle rule for glacier. # The rule name will depend on the S3_PREFIX -- if - # that's empty, then the rule is just "z3-transition"; + # that's empty, then the rule is just "z3 transition"; # otherise, it is "z3 transition ${S3_PREFIX}" (but with # '/' converted to ' '). lifecycle_rule_name = "z3 transition" From 0c92dcd531c2a9b1fc9e55b1bc4949860f3e0387 Mon Sep 17 00:00:00 2001 From: Sean Eric Fagan Date: Sat, 10 Jun 2017 12:42:49 -0700 Subject: [PATCH 3/6] Hack fix for an attribute error. --- z3/snap.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/z3/snap.py b/z3/snap.py index b3570ed..99e135f 100644 --- a/z3/snap.py +++ b/z3/snap.py @@ -405,11 +405,17 @@ def restore(self, snap_name, dry_run=False, force=False): current_snap = self.s3_manager.get(snap_name) if current_snap is None: raise Exception('no such snapshot "{}"'.format(snap_name)) - if current_snap.key.ongoing_restore == True: - raise Exception('snapshot {} is currently being restore from glocier; try again later'.format(snap_name)) - if current_snap.key.storage_class == "GLACIER": - current_snap.key.restore(days=5) - raise Exception('snapshot {} is currently in glacier storage, requesting transfer now'.format(snap_name)) + try: + if current_snap.key.ongoing_restore == True: + raise Exception('snapshot {} is currently being restore from glocier; try again later'.format(snap_name)) + if current_snap.key.storage_class == "GLACIER": + current_snap.key.restore(days=5) + raise Exception('snapshot {} is currently in glacier storage, requesting transfer now'.format(snap_name)) + except AttributeError: + # This seems to be if the FakeKey object doesn't have the ongoing_restore attribute + pass + except: + raise to_restore = [] while True: z_snap = self.zfs_manager.get(current_snap.name) From 6794c2ee60cffd10e218bea25ed325983200ba1e Mon Sep 17 00:00:00 2001 From: Sean Eric Fagan Date: Fri, 16 Jun 2017 09:36:24 -0700 Subject: [PATCH 4/6] Feedback: * Add the glacier-related attributes to FakeKey. * Because of that, don't catch AttributeError. Yay * Don't change the key's storage class in restore if dry_run. * Move the lifecycle management into a function * Related to that, only call if it dry_run is false. * Somewhat-related to that, only create the bucket if the subcommand is backup. This still creates the bucket even if dry run is given. --- _tests/test_snap.py | 3 +- z3/snap.py | 85 ++++++++++++++++++++++----------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/_tests/test_snap.py b/_tests/test_snap.py index 264f1be..3fdfd04 100644 --- a/_tests/test_snap.py +++ b/_tests/test_snap.py @@ -27,7 +27,8 @@ def __init__(self, name, metadata=None): self.key = name self.metadata = metadata self.size = 1234 - + self.storage_class = "STORAGE" + self.ongoing_restore = False class FakeBucket(object): rand_prefix = 'test-' + ''.join([random.choice(string.ascii_letters) for _ in xrange(8)]) + '/' diff --git a/z3/snap.py b/z3/snap.py index 99e135f..b99743d 100644 --- a/z3/snap.py +++ b/z3/snap.py @@ -405,17 +405,12 @@ def restore(self, snap_name, dry_run=False, force=False): current_snap = self.s3_manager.get(snap_name) if current_snap is None: raise Exception('no such snapshot "{}"'.format(snap_name)) - try: + if dry_run is False: if current_snap.key.ongoing_restore == True: raise Exception('snapshot {} is currently being restore from glocier; try again later'.format(snap_name)) if current_snap.key.storage_class == "GLACIER": current_snap.key.restore(days=5) raise Exception('snapshot {} is currently in glacier storage, requesting transfer now'.format(snap_name)) - except AttributeError: - # This seems to be if the FakeKey object doesn't have the ongoing_restore attribute - pass - except: - raise to_restore = [] while True: z_snap = self.zfs_manager.get(current_snap.name) @@ -502,7 +497,13 @@ def list_snapshots(bucket, s3_prefix, filesystem, snapshot_prefix): print(fmt.format(*line)) -def do_backup(bucket, s3_prefix, filesystem, snapshot_prefix, full, snapshot, compressor, dry, parseable): +def do_backup(bucket, s3_prefix, filesystem, snapshot_prefix, full, snapshot, + compressor, dry, parseable, use_glacier=False): + if dry is False: + # This will modify the lifecycle rules for the bucket, + # so we only do it if it's not a dry run. + + glacier_lifecycle(bucket, s3_prefix, use_glacier) prefix = "{}@{}".format(filesystem, snapshot_prefix) s3_mgr = S3SnapshotManager(bucket, s3_prefix=s3_prefix, snapshot_prefix=prefix) zfs_mgr = ZFSSnapshotManager(fs_name=filesystem, snapshot_prefix=snapshot_prefix) @@ -590,33 +591,7 @@ def to_bool(s): subparsers.add_parser('status', help='show status of current backups') return parser.parse_args() - -@handle_soft_errors -def main(): - cfg = get_config() - args = parse_args() - - try: - s3_key_id, s3_secret, bucket_name = cfg['S3_KEY_ID'], cfg['S3_SECRET'], cfg['BUCKET'] - - extra_config = {} - if 'HOST' in cfg: - extra_config['host'] = cfg['HOST'] - except KeyError as err: - sys.stderr.write("Configuration error! {} is not set.\n".format(err)) - sys.exit(1) - - s3 = boto.connect_s3(s3_key_id, s3_secret, **extra_config) - try: - bucket = s3.get_bucket(bucket_name) - except boto.exception.S3ResponseError as e: - if e.error_code == 'NoSuchBucket': - # Let's try creating it - bucket = s3.create_bucket(bucket_name) - print("Created bucket {}: {}".format(bucket_name, bucket), file=sys.stderr) - else: - raise - +def glacier_lifecycle(bucket, s3_prefix, use_glacier): try: lifecycle = bucket.get_lifecycle_config() except boto.exception.S3ResponseError: @@ -628,10 +603,9 @@ def main(): # otherise, it is "z3 transition ${S3_PREFIX}" (but with # '/' converted to ' '). lifecycle_rule_name = "z3 transition" - if args.s3_prefix: - lifecycle_rule_name += " " + args.s3_prefix.replace("/", " ") - while lifecycle_rule_name.endswith(" "): - lifecycle_rule_name = lifecycle_rule_name[:-1] + if s3_prefix: + lifecycle_rule_name += " " + s3_prefix.replace("/", " ") + lifecycle_rule_name = lifecycle_rule_name.rstrip() rule_index = None for indx, rule in enumerate(lifecycle or []): @@ -639,7 +613,7 @@ def main(): rule_index = indx break - if not args.use_glacier: + if not use_glacier: # If we don't use glacier, we want to remove the lifecycle policy # if it exists if rule_index is not None: @@ -650,16 +624,41 @@ def main(): if lifecycle is None: lifecycle = boto.s3.lifecycle.Lifecycle() transition=boto.s3.lifecycle.Transition(days=1, storage_class="GLACIER") - print("trasition rule = {}".format(transition), file=sys.stderr) - print("prefix = {}".format(args.s3_prefix or None), file=sys.stderr) lifecycle.add_rule(id=lifecycle_rule_name, status="Enabled", - prefix=args.s3_prefix or None, + prefix=s3_prefix or None, transition=transition) - print("lifecycle = {}".format(lifecycle.to_xml()), file=sys.stderr) if lifecycle is not None: bucket.configure_lifecycle(lifecycle) + +@handle_soft_errors +def main(): + cfg = get_config() + args = parse_args() + + try: + s3_key_id, s3_secret, bucket_name = cfg['S3_KEY_ID'], cfg['S3_SECRET'], cfg['BUCKET'] + + extra_config = {} + if 'HOST' in cfg: + extra_config['host'] = cfg['HOST'] + except KeyError as err: + sys.stderr.write("Configuration error! {} is not set.\n".format(err)) + sys.exit(1) + + s3 = boto.connect_s3(s3_key_id, s3_secret, **extra_config) + try: + bucket = s3.get_bucket(bucket_name) + except boto.exception.S3ResponseError as e: + if e.error_code == 'NoSuchBucket' and args.subcommand == 'backup': + # Let's try creating it + bucket = s3.create_bucket(bucket_name) + print("Created bucket {}: {}".format(bucket_name, bucket), file=sys.stderr) + else: + raise + glacier_lifecycle(bucket, args.s3_prefix, args.use_glacier) + fs_section = "fs:{}".format(args.filesystem) if args.snapshot_prefix is None: snapshot_prefix = cfg.get("SNAPSHOT_PREFIX", section=fs_section) From 6190c5c71957cd1435acfd8eebb914d1681d8ee4 Mon Sep 17 00:00:00 2001 From: Sean Eric Fagan Date: Fri, 16 Jun 2017 11:09:06 -0700 Subject: [PATCH 5/6] Fix run-time issues. --- z3/snap.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/z3/snap.py b/z3/snap.py index b99743d..e8abb94 100644 --- a/z3/snap.py +++ b/z3/snap.py @@ -502,7 +502,6 @@ def do_backup(bucket, s3_prefix, filesystem, snapshot_prefix, full, snapshot, if dry is False: # This will modify the lifecycle rules for the bucket, # so we only do it if it's not a dry run. - glacier_lifecycle(bucket, s3_prefix, use_glacier) prefix = "{}@{}".format(filesystem, snapshot_prefix) s3_mgr = S3SnapshotManager(bucket, s3_prefix=s3_prefix, snapshot_prefix=prefix) @@ -623,7 +622,7 @@ def glacier_lifecycle(bucket, s3_prefix, use_glacier): # Okay, we need to add a lifecycle if lifecycle is None: lifecycle = boto.s3.lifecycle.Lifecycle() - transition=boto.s3.lifecycle.Transition(days=1, storage_class="GLACIER") + transition=boto.s3.lifecycle.Transition(days=0, storage_class="GLACIER") lifecycle.add_rule(id=lifecycle_rule_name, status="Enabled", prefix=s3_prefix or None, @@ -657,8 +656,6 @@ def main(): else: raise - glacier_lifecycle(bucket, args.s3_prefix, args.use_glacier) - fs_section = "fs:{}".format(args.filesystem) if args.snapshot_prefix is None: snapshot_prefix = cfg.get("SNAPSHOT_PREFIX", section=fs_section) @@ -677,7 +674,8 @@ def main(): do_backup(bucket, s3_prefix=args.s3_prefix, snapshot_prefix=snapshot_prefix, filesystem=args.filesystem, full=args.full, snapshot=args.snapshot, - dry=args.dry, compressor=compressor, parseable=args.parseable) + dry=args.dry, compressor=compressor, parseable=args.parseable, + use_glacier=args.use_glacier) elif args.subcommand == 'restore': restore(bucket, s3_prefix=args.s3_prefix, snapshot_prefix=snapshot_prefix, filesystem=args.filesystem, snapshot=args.snapshot, dry=args.dry, From 0c91eca3f25cbcaf29dd41bcc48016b1247a26fa Mon Sep 17 00:00:00 2001 From: Sean Eric Fagan Date: Fri, 16 Jun 2017 12:23:35 -0700 Subject: [PATCH 6/6] Thanks to review, and experiment, found out that storage class stays "glacier" after an object is restored. If ongoing_restore is None, then the object has not been restored. --- z3/snap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/z3/snap.py b/z3/snap.py index e8abb94..aa45209 100644 --- a/z3/snap.py +++ b/z3/snap.py @@ -408,7 +408,7 @@ def restore(self, snap_name, dry_run=False, force=False): if dry_run is False: if current_snap.key.ongoing_restore == True: raise Exception('snapshot {} is currently being restore from glocier; try again later'.format(snap_name)) - if current_snap.key.storage_class == "GLACIER": + elif current_snap.key.ongoing_restore is None and current_snap.key.storage_class == "GLACIER": current_snap.key.restore(days=5) raise Exception('snapshot {} is currently in glacier storage, requesting transfer now'.format(snap_name)) to_restore = []