First version, mostly working but featureless...

This commit is contained in:
Fredrik Eriksson 2017-05-21 22:33:57 +02:00
parent 0b5992c88c
commit f328cef917
4 changed files with 607 additions and 0 deletions

255
bin/zsnapper Normal file
View File

@ -0,0 +1,255 @@
#!/usr/bin/env python
import datetime
import os
import re
import logging
import logging.handlers
import subprocess
import sys
try:
import configparser
except ImportError:
import ConfigParser as configparser
from string import Template
import zsnaplib
LOGGER = 'zsnapper'
RET_CODES = {
'SUCCESS': 0,
'ERROR': 1,
'FAILED': 2
}
DEFAULT_CONFIG = {
'snapshot_interval': None,
'custom_keep_interval': None,
'keep_yearly': 0,
'keep_monthly': 0,
'keep_weekly': 0,
'keep_daily': 0,
'keep_hourly': 0,
'keep_30min': 0,
'keep_15min': 0,
'keep_5min': 0,
'keep_1min': 0,
'keep_custom': 0,
'remote_enable': False,
'remote_zfs_cmd': None,
'remote_test_cmd': None,
'remote_zfs_target': None,
}
timedelta_regex = re.compile('([0-9]+)([dhm])')
def remote_is_available(conf):
log = logging.getLogger(LOGGER)
cmdstr = Template(conf['remote_test_cmd']).safe_substitute(conf)
cmd = cmdstr.split()
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = proc.communicate()
log.info('Healthcheck "{}" returned {}'.format(cmdstr, proc.returncode))
return proc.returncode == 0
def str_to_timedelta(deltastr):
delta = datetime.timedelta()
for match in timedelta_regex.finditer(deltastr):
if match.group(2) == 'd':
delta += datetime.timedelta(days=int(match.group(1)))
elif match.group(2) == 'h':
delta += datetime.timedelta(hours=int(match.group(1)))
elif match.group(2) == 'm':
delta += datetime.timedelta(minutes=int(match.group(1)))
return delta
def get_config_for_fs(fs, config):
fs_config = DEFAULT_CONFIG
fs_build = ''
for fs_part in fs.split('/'):
fs_build += fs_part
if fs_build in config:
fs_config.update(config[fs_build])
if fs_build == fs:
break
fs_build += '/'
return fs_config
def main():
config = configparser.SafeConfigParser()
config.read('/etc/zsnapper.ini')
sudo = False
ret = RET_CODES['SUCCESS']
log = logging.getLogger(LOGGER)
if os.getuid() != 0:
sudo = True
try:
sudo = config.get('settings', 'sudo')
except (configparser.NoOptionError, configparser.NoSectionError):
pass
fslist = sorted(zsnaplib.get_filesystems(sudo))
snapshots = zsnaplib.get_snapshots(sudo)
now = datetime.datetime.now()
# if we fail to create or send a snapshot we do not want to remove
# the existing snapshots...
failed_snapshots = set()
# First iteration: create snapshots
for fs in fslist:
conf = get_config_for_fs(fs, config)
if not conf['snapshot_interval']:
continue
interval = str_to_timedelta(conf['snapshot_interval'])
if fs in snapshots and snapshots[fs] and snapshots[fs][0]:
last_snap = snapshots[fs][0]
else:
last_snap = datetime.datetime.min
if interval > datetime.timedelta() and last_snap+interval < now:
try:
zsnaplib.create_snapshot(fs, sudo)
log.info('{} snapshot created'.format(fs))
except zsnaplib.ZFSSnapshotError as e:
log.warning(e)
ret = RET_CODES['ERROR']
failed_snapshots.add(fs)
# reload all snapshots so we get our new snapshots here
snapshots = zsnaplib.get_snapshots(sudo)
# Second iteration: Send snapshots
remote_hosts = {}
remote_targets = {}
for fs in fslist:
conf = get_config_for_fs(fs, config)
remote_fslist = None
remote_snapshots = None
if not conf['remote_enable']:
continue
if conf['remote_test_cmd'] and not remote_is_available(conf):
failed_snapshots.add(fs)
continue
remote_fs = conf['remote_zfs_target']
rel_local = [k for k, v in remote_targets.items() if v == remote_fs]
if rel_local:
rel_local = rel_local[0]
rel_fs = fs[len(rel_local):]
remote_fs = '{}{}'.format(remote_fs, rel_fs)
remote_targets[fs] = remote_fs
# Figure out the state of remote zfs
remote_zfs_cmd = Template(conf['remote_zfs_cmd']).safe_substitute(conf)
remote_zfs_cmd = remote_zfs_cmd.split()
# to avoid running too many commands on remote host, save result if we
# know which host we're working with.
if 'remote_host' in conf:
if conf['remote_host'] in remote_hosts:
remote_fslist = remote_hosts[conf['remote_host']]['fslist']
remote_snapshots = remote_hosts[conf['remote_host']]['snapshots']
else:
remote_fslist = zsnaplib.get_filesystems(zfs_cmd=remote_zfs_cmd)
remote_snapshots = zsnaplib.get_snapshots(zfs_cmd=remote_zfs_cmd)
remote_hosts[conf['remote_host']] = {
'fslist': remote_fslist,
'snapshots': remote_snapshots
}
if not remote_fslist:
remote_fslist = zsnaplib.get_filesystems(zfs_cmd=remote_zfs_cmd)
if not remote_snapshots:
remote_snapshots = zsnaplib.get_snapshots(zfs_cmd=remote_zfs_cmd)
remote_zfs_cmd.extend(['receive', remote_fs])
if remote_fs not in remote_snapshots:
# Remote FS doesn't exist, send a new copy
log.info('{} sending base copy to {}'.format(fs, ' '.join(remote_zfs_cmd)))
try:
zsnaplib.send_snapshot(fs, snapshots[fs][0], remote_zfs_cmd, sudo)
log.info('{} base copy sent'.format(fs))
except zsnaplib.ZFSSnapshotError as e:
failed_snapshots.add(fs)
log.warning(e)
ret = RET_CODES['ERROR']
continue
else:
# Remote FS exists, find last common snapshot
last_remote = None
for remote_snap in remote_snapshots[remote_fs]:
if remote_snap in snapshots[fs]:
last_remote = remote_snap
break
if not last_remote:
failed_snapshots.add(fs)
log.warning('No common snapshot local and remote, you need to create a new base copy!')
ret = RET_CODES['ERROR']
continue
last_local = snapshots[fs][0]
if last_remote == last_local:
log.info("{} snapshot from {} is already present on remote".format(fs, last_local))
continue
log.info('{} incremental {} -> {}, remote is {}'.format(fs, last_remote, snapshots[fs][0], ' '.join(remote_zfs_cmd)))
try:
zsnaplib.send_snapshot(fs, snapshots[fs][0], remote_zfs_cmd, sudo, repl_from=last_remote)
log.info('{} successfully sent to remote'.format(fs))
except zsnaplib.ZFSSnapshotError as e:
log.warning(e)
# Third iteration: weed old snapshots
remote_hosts = {}
for fs in fslist:
conf = get_config_for_fs(fs, config)
if fs in failed_snapshots:
log.info("Not weeding {} because of snapshot creation/send failure".format(fs))
continue
if fs not in snapshots:
continue
kwargs = {k: int(v) for k, v in conf.items() if k in [
'keep_custom',
'keep_yearly',
'keep_monthly',
'keep_weekly',
'keep_daily',
'keep_hourly',
'keep_30min',
'keep_15min',
'keep_5min',
'keep_1min']}
if conf['custom_keep_interval']:
kwargs['custom_keep_interval'] = str_to_timedelta(conf['custom_keep_interval'])
kwargs['sudo'] = sudo
zsnaplib.weed_snapshots(
fs,
# do not remove the snapshot just created
snapshots[fs][1:],
**kwargs)
if __name__ == '__main__':
log = logging.getLogger(LOGGER)
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
handler.setLevel(logging.WARNING)
log.addHandler(handler)
handler = logging.handlers.SysLogHandler(address='/dev/log')
handler.setLevel(logging.INFO)
log.addHandler(handler)
sys.exit(main())

34
setup.py Normal file
View File

@ -0,0 +1,34 @@
#!/usr/bin/env python
from os import environ
try:
from setuptools import setup
except ImportError:
from distutils import setup
import pwgen
version = '0.1'
setup(
name='zsnapper',
version=str(version),
description="ZFS snapshot manager",
author="Fredrik Eriksson",
author_email="zsnapper@wb9.se",
url="https://github.com/fredrik-eriksson/zsnapper",
platforms=['any'],
license='BSD',
packages=['zsnaplib'],
classifiers=[
'Development Status :: 1 - Planning',
'Environment :: Console',
'Intended Audience :: System Administrators',
'License :: OSI Approved :: BSD License',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 3',
'Topic :: Utilities',
],
keywords='zfs snapshot backup',
scripts=['bin/zsnapper']
)

273
zsnaplib/__init__.py Normal file
View File

@ -0,0 +1,273 @@
import datetime
import logging
import re
import subprocess
import sys
time_format='%Y-%m-%d_%H%M'
zfs_bin='/sbin/zfs'
sudo_bin='/usr/bin/sudo'
re_snapshot = re.compile(r'^(.*)@([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{4})$')
logger = 'zsnapper'
class ZFSSnapshotError(Exception):
pass
def do_zfs_command(args, sudo, pipecmd=None, zfs_cmd=[zfs_bin]):
cmd = []
sudopw = None
if sudo:
cmd.append(sudo_bin)
if sys.version_info[0] == 3:
if isinstance(sudo, str):
cmd.append('--stdin')
sudopw = '{}\n'.format(sudo)
elif isinstance(sudo, basestring):
cmd.append('--stdin')
sudopw = '{}\n'.format(sudo)
cmd.extend(zfs_cmd)
cmd.extend(args)
proc = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ctrl_proc = proc
if pipecmd:
proc2 = subprocess.Popen(
pipecmd,
stdin=proc.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.stdout.close()
ctrl_proc = proc2
(out, err) = ctrl_proc.communicate()
if ctrl_proc.returncode != 0:
print(proc.returncode)
raise ZFSSnapshotError('Failed to execute {}: {}'.format(cmd, err))
return out
def send_snapshot(fs, snap, recv_cmd, sudo=False, repl_from=None):
snap = snap.strftime(time_format)
pipecmd = recv_cmd
if repl_from:
repl_from = repl_from.strftime(time_format)
args = [ 'send', '-i', repl_from, '{}@{}'.format(fs, snap) ]
else:
args = [ 'send', '{}@{}'.format(fs, snap) ]
do_zfs_command(args, sudo, pipecmd=pipecmd)
def create_snapshot(fs, sudo=False):
d = datetime.datetime.now().strftime(time_format)
args = ['snapshot', '{}@{}'.format(fs, d)]
do_zfs_command(args, sudo)
def get_filesystems(sudo=False, zfs_cmd=[zfs_bin]):
args = ['list', '-H']
out = do_zfs_command(args, sudo, zfs_cmd=zfs_cmd)
ret = set()
for row in out.splitlines():
row = row.decode('UTF-8')
ret.add(row.split()[0])
return ret
def get_snapshots(sudo=False, zfs_cmd=[zfs_bin]):
args = [ 'list', '-H', '-t', 'snapshot' ]
out = do_zfs_command(args, sudo, zfs_cmd=zfs_cmd)
snapshots = {}
for row in out.splitlines():
row = row.decode('UTF-8').split()[0]
res = re_snapshot.match(row)
if res:
d = datetime.datetime.strptime(res.group(2), time_format)
if res.group(1) in snapshots:
snapshots[res.group(1)].append(d)
else:
snapshots[res.group(1)] = [d]
for l in snapshots.values():
l.sort(reverse=True)
return snapshots
def remove_snapshot(fs, date, sudo=False):
date = date.strftime(time_format)
args = [ 'destroy', '{}@{}'.format(fs, date) ]
print("would remove snapshot {}@{}".format(fs, date))
return
do_zfs_command(args, sudo)
def weed_snapshots(
fs,
dates,
custom_keep_interval = None,
keep_custom = 0,
keep_yearly = 0,
keep_monthly = 0,
keep_weekly = 0,
keep_daily = 0,
keep_hourly = 0,
keep_30min = 0,
keep_15min = 0,
keep_5min = 0,
keep_1min = 0,
sudo = False):
log = logging.getLogger(logger)
keep = {
'custom': [],
'year' : [],
'month' : [],
'week' : [],
'day' : [],
'hour' : [],
'min30' : [],
'min15' : [],
'min5' : [],
'min1' : []
}
saved = {
'custom': [],
'year' : [],
'month' : [],
'week' : [],
'day' : [],
'hour' : [],
'min30' : [],
'min15' : [],
'min5' : [],
'min1' : []
}
for date in sorted(dates):
min1 = date-datetime.timedelta(seconds=date.second, microseconds=date.microsecond)
min5 = date-datetime.timedelta(minutes=date.minute%5, seconds=date.second, microseconds=date.microsecond)
min15 = date-datetime.timedelta(minutes=date.minute%15, seconds=date.second, microseconds=date.microsecond)
min30 = date-datetime.timedelta(minutes=date.minute%30, seconds=date.second, microseconds=date.microsecond)
hour = date-datetime.timedelta(minutes=date.minute, seconds=date.second, microseconds=date.microsecond)
day = datetime.datetime.combine(date.date(), datetime.time.min)
week = datetime.datetime.combine(date.date()-datetime.timedelta(days=date.weekday()), datetime.time.min)
month = datetime.datetime(year=date.year, month=date.month, day=1)
year = datetime.datetime(year=date.year, month=1, day=1)
# yearly snapshots
if year not in saved['year']:
saved['year'].append(year)
keep['year'].append(date)
if month not in saved['month']:
saved['month'].append(month)
keep['month'].append(date)
if week not in saved['week']:
saved['week'].append(week)
keep['week'].append(date)
if day not in saved['day']:
saved['day'].append(day)
keep['day'].append(date)
if hour not in saved['hour']:
saved['hour'].append(hour)
keep['hour'].append(date)
if min30 not in saved['min30']:
saved['min30'].append(min30)
keep['min30'].append(date)
if min15 not in saved['min15']:
saved['min15'].append(min15)
keep['min15'].append(date)
if min5 not in saved['min5']:
saved['min5'].append(min5)
keep['min5'].append(date)
if min1 not in saved['min1']:
saved['min1'].append(min1)
keep['min1'].append(date)
if custom_keep_interval:
cur = year
while cur+custom_keep_interval < date:
cur += custom_keep_interval
if cur not in saved['custom']:
saved['custom'].append(cur)
keep['custom'].append(date)
if keep_yearly:
saved['year'] = saved['year'][-keep_yearly:]
else:
saved['year'] = []
if keep_monthly:
saved['month'] = saved['month'][-keep_monthly:]
else:
saved['month'] = []
if keep_weekly:
saved['week'] = saved['week'][-keep_weekly:]
else:
saved['week'] = []
if keep_daily:
saved['day'] = saved['day'][-keep_daily:]
else:
saved['day'] = []
if keep_hourly:
saved['hour'] = saved['hour'][-keep_hourly:]
else:
saved['hour'] = []
if keep_30min:
saved['min30'] = saved['min30'][-keep_30min:]
else:
saved['min30'] = []
if keep_15min:
saved['min15'] = saved['min15'][-keep_15min:]
else:
saved['min15'] = []
if keep_5min:
saved['min5'] = saved['min5'][-keep_5min:]
else:
saved['min5'] = []
if keep_1min:
saved['min1'] = saved['min1'][-keep_1min:]
else:
saved['min1'] = []
if keep_custom:
saved['custom'] = saved['custom'][-keep_custom:]
else:
saved['custom'] = []
all_saved = []
all_saved.extend(saved['year'])
all_saved.extend(saved['month'])
all_saved.extend(saved['week'])
all_saved.extend(saved['day'])
all_saved.extend(saved['hour'])
all_saved.extend(saved['min30'])
all_saved.extend(saved['min15'])
all_saved.extend(saved['min5'])
all_saved.extend(saved['min1'])
all_saved.extend(saved['custom'])
all_saved = set(all_saved)
to_remove = [date for date in dates if date not in all_saved]
for date in to_remove:
try:
log.info('{}: removing snapshot from {}'.format(fs, date))
remove_snapshot(fs, date, sudo=sudo)
except ZFSSnapshotError as e:
log.error(str(e))

45
zsnapper.ini-sample Normal file
View File

@ -0,0 +1,45 @@
[settings]
# NOTE:
# --stdin is used to pass password to sudo, this does not work the first time
# a user uses sudo, so make sure to run a sudo command manually first
#
# or preferably, leave this commented out and use NOPASSWD for sudo...
;sudo=<password>
[tank]
snapshot_interval=1h
# NOTE:
# The command arguments must not contain whitespace characters, since
# split() is used to create an array to subprocess.Popen()
remote_zfs_cmd=/usr/bin/ssh ${remote_user}@${remote_host} /usr/bin/sudo /sbin/zfs
remote_test_cmd=/usr/bin/ssh ${remote_user}@${remote_host} echo "success"
remote_user=backup
remote_host=hem.winterbird.org
remote_zfs_target=tank/backup/asuna/tank
# NOTE:
# should be empty or 0 for negative value
remote_enable=1
keep_hourly=24
keep_weekly=4
keep_monthly=4
[tank/SWAP]
snapshot_interval=
remote_enable=
[tank/media]
snapshot_interval=15m
[tank/tmp]
snapshot_interval=
remote_enable=
[tank/var/log]
snapshot_interval=1m
[tank/var/tmp]
snapshot_interval=
remote_enable=