First version, mostly working but featureless...

bin/zsnapper

@@ -0,0 +1,255 @@
+#!/usr/bin/env python
+
+import datetime
+import os
+import re
+import logging
+import logging.handlers
+import subprocess
+import sys
+
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+from string import Template
+
+import zsnaplib
+
+LOGGER = 'zsnapper'
+
+RET_CODES = {
+        'SUCCESS': 0,
+        'ERROR': 1,
+        'FAILED': 2
+        }
+
+DEFAULT_CONFIG = {
+        'snapshot_interval': None,
+        'custom_keep_interval': None,
+        'keep_yearly': 0,
+        'keep_monthly': 0,
+        'keep_weekly': 0,
+        'keep_daily': 0,
+        'keep_hourly': 0,
+        'keep_30min': 0,
+        'keep_15min': 0,
+        'keep_5min': 0,
+        'keep_1min': 0,
+        'keep_custom': 0,
+        'remote_enable': False,
+        'remote_zfs_cmd': None,
+        'remote_test_cmd': None,
+        'remote_zfs_target': None,
+        }
+
+timedelta_regex = re.compile('([0-9]+)([dhm])')
+
+def remote_is_available(conf):
+    log = logging.getLogger(LOGGER)
+    cmdstr = Template(conf['remote_test_cmd']).safe_substitute(conf)
+    cmd = cmdstr.split()
+    proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+    (out, err) = proc.communicate()
+
+    log.info('Healthcheck "{}" returned {}'.format(cmdstr, proc.returncode))
+    return proc.returncode == 0
+
+
+def str_to_timedelta(deltastr):
+
+    delta = datetime.timedelta()
+    for match in timedelta_regex.finditer(deltastr):
+        if match.group(2) == 'd':
+            delta += datetime.timedelta(days=int(match.group(1)))
+        elif match.group(2) == 'h':
+            delta += datetime.timedelta(hours=int(match.group(1)))
+        elif match.group(2) == 'm':
+            delta += datetime.timedelta(minutes=int(match.group(1)))
+    return delta
+
+def get_config_for_fs(fs, config):
+    fs_config = DEFAULT_CONFIG
+    fs_build = ''
+    for fs_part in fs.split('/'):
+        fs_build += fs_part
+        if fs_build in config:
+            fs_config.update(config[fs_build])
+        if fs_build == fs:
+            break
+        fs_build += '/'
+
+    return fs_config
+
+def main():
+    config = configparser.SafeConfigParser()
+    config.read('/etc/zsnapper.ini')
+    sudo = False
+    ret = RET_CODES['SUCCESS']
+    log = logging.getLogger(LOGGER)
+
+    if os.getuid() != 0:
+        sudo = True
+        try:
+            sudo = config.get('settings', 'sudo')
+        except (configparser.NoOptionError, configparser.NoSectionError):
+            pass
+    
+    fslist = sorted(zsnaplib.get_filesystems(sudo))
+    snapshots = zsnaplib.get_snapshots(sudo)
+    now = datetime.datetime.now()
+    
+    # if we fail to create or send a snapshot we do not want to remove 
+    # the existing snapshots...
+    failed_snapshots = set()
+
+    # First iteration: create snapshots
+    for fs in fslist:
+        conf = get_config_for_fs(fs, config)
+        if not conf['snapshot_interval']:
+            continue
+
+        interval = str_to_timedelta(conf['snapshot_interval'])
+        if fs in snapshots and snapshots[fs] and snapshots[fs][0]:
+            last_snap = snapshots[fs][0]
+        else:
+            last_snap = datetime.datetime.min
+        if interval > datetime.timedelta() and last_snap+interval < now:
+            try:
+                zsnaplib.create_snapshot(fs, sudo)
+                log.info('{} snapshot created'.format(fs))
+            except zsnaplib.ZFSSnapshotError as e:
+                log.warning(e)
+                ret = RET_CODES['ERROR']
+                failed_snapshots.add(fs)
+        
+    # reload all snapshots so we get our new snapshots here
+    snapshots = zsnaplib.get_snapshots(sudo)
+
+    # Second iteration: Send snapshots
+    remote_hosts = {}
+    remote_targets = {}
+    for fs in fslist:
+        conf = get_config_for_fs(fs, config)
+        remote_fslist = None
+        remote_snapshots = None
+        if not conf['remote_enable']:
+            continue
+        if conf['remote_test_cmd'] and not remote_is_available(conf):
+            failed_snapshots.add(fs)
+            continue
+
+        remote_fs = conf['remote_zfs_target']
+        rel_local = [k for k, v in remote_targets.items() if v == remote_fs]
+        if rel_local:
+            rel_local = rel_local[0]
+            rel_fs = fs[len(rel_local):]
+            remote_fs = '{}{}'.format(remote_fs, rel_fs)
+        remote_targets[fs] = remote_fs
+
+        # Figure out the state of remote zfs
+        remote_zfs_cmd = Template(conf['remote_zfs_cmd']).safe_substitute(conf)
+        remote_zfs_cmd = remote_zfs_cmd.split()
+        # to avoid running too many commands on remote host, save result if we
+        # know which host we're working with.
+        if 'remote_host' in conf:
+            if conf['remote_host'] in remote_hosts:
+                remote_fslist = remote_hosts[conf['remote_host']]['fslist']
+                remote_snapshots = remote_hosts[conf['remote_host']]['snapshots']
+            else:
+                remote_fslist = zsnaplib.get_filesystems(zfs_cmd=remote_zfs_cmd)
+                remote_snapshots = zsnaplib.get_snapshots(zfs_cmd=remote_zfs_cmd)
+                remote_hosts[conf['remote_host']] = {
+                        'fslist': remote_fslist,
+                        'snapshots': remote_snapshots
+                        }
+        if not remote_fslist:
+            remote_fslist = zsnaplib.get_filesystems(zfs_cmd=remote_zfs_cmd)
+        if not remote_snapshots:
+            remote_snapshots = zsnaplib.get_snapshots(zfs_cmd=remote_zfs_cmd)
+
+        remote_zfs_cmd.extend(['receive', remote_fs])
+
+        if remote_fs not in remote_snapshots:
+            # Remote FS doesn't exist, send a new copy
+            log.info('{} sending base copy to {}'.format(fs, ' '.join(remote_zfs_cmd)))
+            try:
+                zsnaplib.send_snapshot(fs, snapshots[fs][0], remote_zfs_cmd, sudo)
+                log.info('{} base copy sent'.format(fs))
+            except zsnaplib.ZFSSnapshotError as e:
+                failed_snapshots.add(fs)
+                log.warning(e)
+                ret = RET_CODES['ERROR']
+                continue
+        else: 
+            # Remote FS exists, find last common snapshot
+            last_remote = None
+            for remote_snap in remote_snapshots[remote_fs]:
+                if remote_snap in snapshots[fs]:
+                    last_remote = remote_snap
+                    break
+            if not last_remote:
+                failed_snapshots.add(fs)
+                log.warning('No common snapshot local and remote, you need to create a new base copy!')
+                ret = RET_CODES['ERROR']
+                continue
+            last_local = snapshots[fs][0]
+            if last_remote == last_local:
+                log.info("{} snapshot from {} is already present on remote".format(fs, last_local))
+                continue
+
+            log.info('{} incremental {} -> {}, remote is {}'.format(fs, last_remote, snapshots[fs][0], ' '.join(remote_zfs_cmd)))
+            try:
+                zsnaplib.send_snapshot(fs, snapshots[fs][0], remote_zfs_cmd, sudo, repl_from=last_remote)
+                log.info('{} successfully sent to remote'.format(fs))
+            except zsnaplib.ZFSSnapshotError as e:
+                log.warning(e)
+
+    # Third iteration: weed old snapshots
+    remote_hosts = {}
+    for fs in fslist:
+        conf = get_config_for_fs(fs, config)
+        if fs in failed_snapshots:
+            log.info("Not weeding {} because of snapshot creation/send failure".format(fs))
+            continue
+        if fs not in snapshots:
+            continue
+
+        kwargs = {k: int(v) for k, v in conf.items() if k in [
+                'keep_custom',
+                'keep_yearly',
+                'keep_monthly',
+                'keep_weekly',
+                'keep_daily',
+                'keep_hourly',
+                'keep_30min',
+                'keep_15min',
+                'keep_5min',
+                'keep_1min']}
+        if conf['custom_keep_interval']:
+            kwargs['custom_keep_interval'] = str_to_timedelta(conf['custom_keep_interval'])
+        kwargs['sudo'] = sudo
+
+        zsnaplib.weed_snapshots(
+                fs,
+                # do not remove the snapshot just created
+                snapshots[fs][1:],
+                **kwargs)
+
+
+
+
+if __name__ == '__main__':
+    log = logging.getLogger(LOGGER)
+    log.setLevel(logging.INFO)
+    handler = logging.StreamHandler()
+    handler.setLevel(logging.WARNING)
+    log.addHandler(handler)
+    handler = logging.handlers.SysLogHandler(address='/dev/log')
+    handler.setLevel(logging.INFO)
+    log.addHandler(handler)
+    sys.exit(main())

setup.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+from os import environ
+
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils import setup
+
+import pwgen
+
+version = '0.1'
+
+setup(
+        name='zsnapper',
+        version=str(version),
+        description="ZFS snapshot manager",
+        author="Fredrik Eriksson",
+        author_email="zsnapper@wb9.se",
+        url="https://github.com/fredrik-eriksson/zsnapper",
+        platforms=['any'],
+        license='BSD',
+        packages=['zsnaplib'],
+        classifiers=[
+            'Development Status :: 1 - Planning',
+            'Environment :: Console',
+            'Intended Audience :: System Administrators',
+            'License :: OSI Approved :: BSD License',
+            'Programming Language :: Python :: 2',
+            'Programming Language :: Python :: 3',
+            'Topic :: Utilities',
+            ],
+        keywords='zfs snapshot backup',
+        scripts=['bin/zsnapper']
+        )

zsnaplib/__init__.py

@@ -0,0 +1,273 @@
+import datetime
+import logging
+import re
+import subprocess
+import sys
+
+time_format='%Y-%m-%d_%H%M'
+zfs_bin='/sbin/zfs'
+sudo_bin='/usr/bin/sudo'
+re_snapshot = re.compile(r'^(.*)@([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{4})$')
+logger = 'zsnapper'
+
+class ZFSSnapshotError(Exception):
+    pass
+
+def do_zfs_command(args, sudo, pipecmd=None, zfs_cmd=[zfs_bin]):
+    cmd = []
+    sudopw = None
+    if sudo:
+        cmd.append(sudo_bin)
+        if sys.version_info[0] == 3:
+            if isinstance(sudo, str):
+                cmd.append('--stdin')
+                sudopw = '{}\n'.format(sudo)
+        elif isinstance(sudo, basestring):
+            cmd.append('--stdin')
+            sudopw = '{}\n'.format(sudo)
+
+    cmd.extend(zfs_cmd)
+    cmd.extend(args)
+    proc = subprocess.Popen(
+            cmd,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+    ctrl_proc = proc
+    if pipecmd:
+        proc2 = subprocess.Popen(
+                pipecmd,
+                stdin=proc.stdout,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+        proc.stdout.close()
+        ctrl_proc = proc2
+
+    (out, err) = ctrl_proc.communicate()
+
+    if ctrl_proc.returncode != 0:
+        print(proc.returncode)
+        raise ZFSSnapshotError('Failed to execute {}: {}'.format(cmd, err))
+    return out
+
+def send_snapshot(fs, snap, recv_cmd, sudo=False, repl_from=None):
+    snap = snap.strftime(time_format)
+    pipecmd = recv_cmd
+    if repl_from:
+        repl_from = repl_from.strftime(time_format)
+        args = [ 'send', '-i', repl_from, '{}@{}'.format(fs, snap) ]
+    else:
+        args = [ 'send', '{}@{}'.format(fs, snap) ]
+
+    do_zfs_command(args, sudo, pipecmd=pipecmd)
+
+
+def create_snapshot(fs, sudo=False):
+
+    d = datetime.datetime.now().strftime(time_format)
+    args = ['snapshot', '{}@{}'.format(fs, d)]
+    do_zfs_command(args, sudo)
+
+def get_filesystems(sudo=False, zfs_cmd=[zfs_bin]):
+    args = ['list', '-H']
+    out = do_zfs_command(args, sudo, zfs_cmd=zfs_cmd)
+    ret = set()
+
+    for row in out.splitlines():
+        row = row.decode('UTF-8')
+        ret.add(row.split()[0])
+    return ret
+
+
+def get_snapshots(sudo=False, zfs_cmd=[zfs_bin]):
+    args = [ 'list', '-H', '-t', 'snapshot' ]
+    out = do_zfs_command(args, sudo, zfs_cmd=zfs_cmd)
+    snapshots = {}
+
+    for row in out.splitlines():
+        row = row.decode('UTF-8').split()[0]
+        res = re_snapshot.match(row)
+        if res:
+            d = datetime.datetime.strptime(res.group(2), time_format)
+            if res.group(1) in snapshots:
+                snapshots[res.group(1)].append(d)
+            else:
+                snapshots[res.group(1)] = [d]
+
+    for l in snapshots.values():
+        l.sort(reverse=True)
+
+    return snapshots
+
+
+def remove_snapshot(fs, date, sudo=False):
+    date = date.strftime(time_format)
+    args = [ 'destroy', '{}@{}'.format(fs, date) ]
+    print("would remove snapshot {}@{}".format(fs, date))
+    return
+    do_zfs_command(args, sudo)
+
+
+def weed_snapshots(
+        fs,
+        dates,
+        custom_keep_interval = None,
+        keep_custom = 0,
+        keep_yearly = 0,
+        keep_monthly = 0,
+        keep_weekly = 0,
+        keep_daily = 0,
+        keep_hourly = 0,
+        keep_30min = 0,
+        keep_15min = 0,
+        keep_5min = 0,
+        keep_1min = 0,
+        sudo = False):
+
+    log = logging.getLogger(logger)
+
+    keep = {
+            'custom': [],
+            'year' : [],
+            'month' : [],
+            'week' : [],
+            'day' : [],
+            'hour' : [],
+            'min30' : [],
+            'min15' : [],
+            'min5' : [],
+            'min1' : []
+            }
+    saved = {
+            'custom': [],
+            'year' : [],
+            'month' : [],
+            'week' : [],
+            'day' : [],
+            'hour' : [],
+            'min30' : [],
+            'min15' : [],
+            'min5' : [],
+            'min1' : []
+            }
+
+    for date in sorted(dates):
+        min1 = date-datetime.timedelta(seconds=date.second, microseconds=date.microsecond)
+        min5 = date-datetime.timedelta(minutes=date.minute%5, seconds=date.second, microseconds=date.microsecond)
+        min15 = date-datetime.timedelta(minutes=date.minute%15, seconds=date.second, microseconds=date.microsecond)
+        min30 = date-datetime.timedelta(minutes=date.minute%30, seconds=date.second, microseconds=date.microsecond)
+        hour = date-datetime.timedelta(minutes=date.minute, seconds=date.second, microseconds=date.microsecond)
+        day = datetime.datetime.combine(date.date(), datetime.time.min)
+        week = datetime.datetime.combine(date.date()-datetime.timedelta(days=date.weekday()), datetime.time.min)
+        month = datetime.datetime(year=date.year, month=date.month, day=1)
+        year = datetime.datetime(year=date.year, month=1, day=1)
+        # yearly snapshots
+        if year not in saved['year']:
+            saved['year'].append(year)
+            keep['year'].append(date)
+        if month not in saved['month']:
+            saved['month'].append(month)
+            keep['month'].append(date)
+        if week not in saved['week']:
+            saved['week'].append(week)
+            keep['week'].append(date)
+        if day not in saved['day']:
+            saved['day'].append(day)
+            keep['day'].append(date)
+        if hour not in saved['hour']:
+            saved['hour'].append(hour)
+            keep['hour'].append(date)
+        if min30 not in saved['min30']:
+            saved['min30'].append(min30)
+            keep['min30'].append(date)
+        if min15 not in saved['min15']:
+            saved['min15'].append(min15)
+            keep['min15'].append(date)
+        if min5 not in saved['min5']:
+            saved['min5'].append(min5)
+            keep['min5'].append(date)
+        if min1 not in saved['min1']:
+            saved['min1'].append(min1)
+            keep['min1'].append(date)
+
+        if custom_keep_interval:
+            cur = year
+            while cur+custom_keep_interval < date:
+                cur += custom_keep_interval
+            if cur not in saved['custom']:
+                saved['custom'].append(cur)
+                keep['custom'].append(date)
+
+    if keep_yearly:
+        saved['year'] = saved['year'][-keep_yearly:]
+    else:
+        saved['year'] = []
+    
+    if keep_monthly:
+        saved['month'] = saved['month'][-keep_monthly:]
+    else:
+        saved['month'] = []
+
+    if keep_weekly:
+        saved['week'] = saved['week'][-keep_weekly:]
+    else:
+        saved['week'] = []
+
+    if keep_daily:
+        saved['day'] = saved['day'][-keep_daily:]
+    else:
+        saved['day'] = []
+
+    if keep_hourly:
+        saved['hour'] = saved['hour'][-keep_hourly:]
+    else:
+        saved['hour'] = []
+
+    if keep_30min:
+        saved['min30'] = saved['min30'][-keep_30min:]
+    else:
+        saved['min30'] = []
+
+    if keep_15min:
+        saved['min15'] = saved['min15'][-keep_15min:]
+    else:
+        saved['min15'] = []
+
+    if keep_5min:
+        saved['min5'] = saved['min5'][-keep_5min:]
+    else:
+        saved['min5'] = []
+
+    if keep_1min:
+        saved['min1'] = saved['min1'][-keep_1min:]
+    else:
+        saved['min1'] = []
+
+    if keep_custom:
+        saved['custom'] = saved['custom'][-keep_custom:]
+    else:
+        saved['custom'] = []
+
+    all_saved = []
+    all_saved.extend(saved['year'])
+    all_saved.extend(saved['month'])
+    all_saved.extend(saved['week'])
+    all_saved.extend(saved['day'])
+    all_saved.extend(saved['hour'])
+    all_saved.extend(saved['min30'])
+    all_saved.extend(saved['min15'])
+    all_saved.extend(saved['min5'])
+    all_saved.extend(saved['min1'])
+    all_saved.extend(saved['custom'])
+    all_saved = set(all_saved)
+
+    to_remove = [date for date in dates if date not in all_saved]
+    for date in to_remove:
+        try:
+            log.info('{}: removing snapshot from {}'.format(fs, date))
+            remove_snapshot(fs, date, sudo=sudo)
+        except ZFSSnapshotError as e:
+            log.error(str(e))
+
+
+

zsnapper.ini-sample

@@ -0,0 +1,45 @@
+[settings]
+# NOTE:
+# --stdin is used to pass password to sudo, this does not work the first time
+# a user uses sudo, so make sure to run a sudo command manually first
+#
+# or preferably, leave this commented out and use NOPASSWD for sudo...
+;sudo=<password>
+
+
+[tank]
+snapshot_interval=1h
+# NOTE:
+# The command arguments must not contain whitespace characters, since
+# split() is used to create an array to subprocess.Popen()
+remote_zfs_cmd=/usr/bin/ssh ${remote_user}@${remote_host} /usr/bin/sudo /sbin/zfs
+remote_test_cmd=/usr/bin/ssh ${remote_user}@${remote_host} echo "success"
+remote_user=backup
+remote_host=hem.winterbird.org
+remote_zfs_target=tank/backup/asuna/tank
+# NOTE:
+# should be empty or 0 for negative value
+remote_enable=1
+
+keep_hourly=24
+keep_weekly=4
+keep_monthly=4
+
+
+[tank/SWAP]
+snapshot_interval=
+remote_enable=
+
+[tank/media]
+snapshot_interval=15m
+
+[tank/tmp]
+snapshot_interval=
+remote_enable=
+
+[tank/var/log]
+snapshot_interval=1m
+
+[tank/var/tmp]
+snapshot_interval=
+remote_enable=