From 8617e3c0c602f29eec7f2b42ffd7cbe7cbbb2ce7 Mon Sep 17 00:00:00 2001 From: Fredrik Eriksson Date: Mon, 22 May 2017 21:25:51 +0200 Subject: [PATCH] * added option to send all or just latest snapshot to remote * added options for zfs send and zfs receive flags --- bin/zsnapper | 94 +++++++++++++++++++++++++++----------------- zsnaplib/__init__.py | 23 ++++++++--- zsnapper.ini-sample | 16 ++++++-- 3 files changed, 88 insertions(+), 45 deletions(-) diff --git a/bin/zsnapper b/bin/zsnapper index 40fa5ce..81dc76d 100644 --- a/bin/zsnapper +++ b/bin/zsnapper @@ -39,6 +39,8 @@ DEFAULT_CONFIG = { 'keep_1min': 0, 'keep_custom': 0, 'remote_enable': False, + 'remote_send_flags': '', + 'remote_recv_flags': '', 'remote_zfs_cmd': None, 'remote_test_cmd': None, 'remote_zfs_target': None, @@ -135,7 +137,6 @@ def main(): remote_targets = {} for fs in fslist: conf = get_config_for_fs(fs, config) - remote_fslist = None remote_snapshots = None if not conf['remote_enable']: continue @@ -143,7 +144,15 @@ def main(): failed_snapshots.add(fs) continue + repl_mode = conf['remote_enable'] remote_fs = conf['remote_zfs_target'] + send_opts = [] + recv_opts = [] + if conf['remote_send_flags']: + send_opts = conf['remote_send_flags'].split() + if conf['remote_recv_flags']: + recv_opts = conf['remote_recv_flags'].split() + rel_local = [k for k, v in remote_targets.items() if v == remote_fs] if rel_local: rel_local = rel_local[0] @@ -158,59 +167,70 @@ def main(): # know which host we're working with. if 'remote_host' in conf: if conf['remote_host'] in remote_hosts: - remote_fslist = remote_hosts[conf['remote_host']]['fslist'] - remote_snapshots = remote_hosts[conf['remote_host']]['snapshots'] + remote_snapshots = remote_hosts[conf['remote_host']] else: - remote_fslist = zsnaplib.get_filesystems(zfs_cmd=remote_zfs_cmd) remote_snapshots = zsnaplib.get_snapshots(zfs_cmd=remote_zfs_cmd) - remote_hosts[conf['remote_host']] = { - 'fslist': remote_fslist, - 'snapshots': remote_snapshots - } - if not remote_fslist: - remote_fslist = zsnaplib.get_filesystems(zfs_cmd=remote_zfs_cmd) + remote_hosts[conf['remote_host']] = remote_snapshots if not remote_snapshots: remote_snapshots = zsnaplib.get_snapshots(zfs_cmd=remote_zfs_cmd) - remote_zfs_cmd.extend(['receive', remote_fs]) - if remote_fs not in remote_snapshots: # Remote FS doesn't exist, send a new copy log.info('{} sending base copy to {}'.format(fs, ' '.join(remote_zfs_cmd))) + # oldest snapshot is base_snap if repl_mode != latest + base_snap = snapshots[fs][-1] + if repl_mode == 'latest': + base_snap = snapshots[fs][0] try: - zsnaplib.send_snapshot(fs, snapshots[fs][0], remote_zfs_cmd, sudo) + zsnaplib.send_snapshot( + fs, + base_snap, + remote_zfs_cmd, + remote_fs, + sudo=sudo, + send_opts=send_opts, + recv_opts=recv_opts) log.info('{} base copy sent'.format(fs)) except zsnaplib.ZFSSnapshotError as e: failed_snapshots.add(fs) log.warning(e) ret = RET_CODES['ERROR'] continue - else: - # Remote FS exists, find last common snapshot - last_remote = None - for remote_snap in remote_snapshots[remote_fs]: - if remote_snap in snapshots[fs]: - last_remote = remote_snap - break - if not last_remote: - failed_snapshots.add(fs) - log.warning('No common snapshot local and remote, you need to create a new base copy!') - ret = RET_CODES['ERROR'] - continue - last_local = snapshots[fs][0] - if last_remote == last_local: - log.info("{} snapshot from {} is already present on remote".format(fs, last_local)) - continue + remote_snapshots[remote_fs] = [base_snap] - log.info('{} incremental {} -> {}, remote is {}'.format(fs, last_remote, snapshots[fs][0], ' '.join(remote_zfs_cmd))) - try: - zsnaplib.send_snapshot(fs, snapshots[fs][0], remote_zfs_cmd, sudo, repl_from=last_remote) - log.info('{} successfully sent to remote'.format(fs)) - except zsnaplib.ZFSSnapshotError as e: - log.warning(e) + # Remote FS now exists, one way or another find last common snapshot + last_remote = None + for remote_snap in remote_snapshots[remote_fs]: + if remote_snap in snapshots[fs]: + last_remote = remote_snap + break + if not last_remote: + failed_snapshots.add(fs) + log.warning('No common snapshot local and remote, you need to create a new base copy!') + ret = RET_CODES['ERROR'] + continue + last_local = snapshots[fs][0] + if last_remote == last_local: + log.info("{} snapshot from {} is already present on remote".format(fs, last_local)) + continue + + log.info('{} incremental {} -> {}, remote is {}'.format(fs, last_remote, snapshots[fs][0], ' '.join(remote_zfs_cmd))) + try: + zsnaplib.send_snapshot( + fs, + snapshots[fs][0], + remote_zfs_cmd, + remote_fs, + sudo=sudo, + send_opts=send_opts, + recv_opts=recv_opts, + repl_from=last_remote, + repl_mode=repl_mode) + log.info('{} successfully sent to remote'.format(fs)) + except zsnaplib.ZFSSnapshotError as e: + log.warning(e) # Third iteration: weed old snapshots - remote_hosts = {} for fs in fslist: conf = get_config_for_fs(fs, config) if fs in failed_snapshots: @@ -236,7 +256,7 @@ def main(): zsnaplib.weed_snapshots( fs, - # do not remove the snapshot just created + # never remove the latest snapshot snapshots[fs][1:], **kwargs) diff --git a/zsnaplib/__init__.py b/zsnaplib/__init__.py index a3e57a9..5efc006 100644 --- a/zsnaplib/__init__.py +++ b/zsnaplib/__init__.py @@ -50,15 +50,30 @@ def do_zfs_command(args, sudo, pipecmd=None, zfs_cmd=[zfs_bin]): raise ZFSSnapshotError('Failed to execute {}: {}'.format(cmd, err)) return out -def send_snapshot(fs, snap, recv_cmd, sudo=False, repl_from=None): +def send_snapshot( + fs, + snap, + remote_zfs_cmd, + remote_target, + sudo=False, + send_opts=[], + recv_opts=[], + repl_mode='all', + repl_from=None): snap = snap.strftime(time_format) - pipecmd = recv_cmd if repl_from: + if repl_mode == 'latest': + inc_flag = '-i' + else: + inc_flag = '-I' + repl_from = repl_from.strftime(time_format) - args = [ 'send', '-i', repl_from, '{}@{}'.format(fs, snap) ] + args = [ 'send' ] + send_opts + [ inc_flag, '{}@{}'.format(fs, repl_from), '{}@{}'.format(fs, snap) ] else: args = [ 'send', '{}@{}'.format(fs, snap) ] + pipecmd = remote_zfs_cmd + [ 'receive' ] + recv_opts + [ remote_target ] + do_zfs_command(args, sudo, pipecmd=pipecmd) @@ -103,8 +118,6 @@ def get_snapshots(sudo=False, zfs_cmd=[zfs_bin]): def remove_snapshot(fs, date, sudo=False): date = date.strftime(time_format) args = [ 'destroy', '{}@{}'.format(fs, date) ] - print("would remove snapshot {}@{}".format(fs, date)) - return do_zfs_command(args, sudo) diff --git a/zsnapper.ini-sample b/zsnapper.ini-sample index c90e488..a29e038 100644 --- a/zsnapper.ini-sample +++ b/zsnapper.ini-sample @@ -8,7 +8,13 @@ [tank] +# frequency of snapshots snapshot_interval=1h + +# Remote replication +# possible other value is 'latest' to only sync the last available snapshot +remote_enable=all + # NOTE: # The command arguments must not contain whitespace characters, since # split() is used to create an array to subprocess.Popen() @@ -17,11 +23,13 @@ remote_test_cmd=/usr/bin/ssh ${remote_user}@${remote_host} echo "success" remote_user=backup remote_host=hem.winterbird.org remote_zfs_target=tank/backup/asuna/tank -# NOTE: -# should be empty or 0 for negative value -remote_enable=1 +# These can be set to use custom arguments to zfs send and zfs receive +; remote_send_flags=-D -p +remote_send_flags= +remote_recv_flags= keep_hourly=24 +keep_daily=7 keep_weekly=4 keep_monthly=4 @@ -39,6 +47,8 @@ remote_enable= [tank/var/log] snapshot_interval=1m +keep_1min=5 +keep_15min=4 [tank/var/tmp] snapshot_interval=