* fixed weeding
* added weed_enable option and default it to unset to prevent removing snapshots of unmanaged file systems * added locking when sending and weeding snapshots * fixed syslog logging format
This commit is contained in:
Fredrik Eriksson
parent
8617e3c0c6
commit
60095c48d2
101
bin/zsnapper
101
bin/zsnapper
@ -28,6 +28,7 @@ RET_CODES = {
|
||||
DEFAULT_CONFIG = {
|
||||
'snapshot_interval': None,
|
||||
'custom_keep_interval': None,
|
||||
'weed_enable': False,
|
||||
'keep_yearly': 0,
|
||||
'keep_monthly': 0,
|
||||
'keep_weekly': 0,
|
||||
@ -87,29 +88,12 @@ def get_config_for_fs(fs, config):
|
||||
|
||||
return fs_config
|
||||
|
||||
def main():
|
||||
config = configparser.SafeConfigParser()
|
||||
config.read('/etc/zsnapper.ini')
|
||||
sudo = False
|
||||
ret = RET_CODES['SUCCESS']
|
||||
|
||||
def do_snapshots(fslist, snapshots, config, sudo):
|
||||
failed_snapshots = set()
|
||||
now = datetime.datetime.now()
|
||||
log = logging.getLogger(LOGGER)
|
||||
|
||||
if os.getuid() != 0:
|
||||
sudo = True
|
||||
try:
|
||||
sudo = config.get('settings', 'sudo')
|
||||
except (configparser.NoOptionError, configparser.NoSectionError):
|
||||
pass
|
||||
|
||||
fslist = sorted(zsnaplib.get_filesystems(sudo))
|
||||
snapshots = zsnaplib.get_snapshots(sudo)
|
||||
now = datetime.datetime.now()
|
||||
|
||||
# if we fail to create or send a snapshot we do not want to remove
|
||||
# the existing snapshots...
|
||||
failed_snapshots = set()
|
||||
|
||||
# First iteration: create snapshots
|
||||
for fs in fslist:
|
||||
conf = get_config_for_fs(fs, config)
|
||||
if not conf['snapshot_interval']:
|
||||
@ -126,15 +110,14 @@ def main():
|
||||
log.info('{} snapshot created'.format(fs))
|
||||
except zsnaplib.ZFSSnapshotError as e:
|
||||
log.warning(e)
|
||||
ret = RET_CODES['ERROR']
|
||||
failed_snapshots.add(fs)
|
||||
|
||||
# reload all snapshots so we get our new snapshots here
|
||||
snapshots = zsnaplib.get_snapshots(sudo)
|
||||
return failed_snapshots
|
||||
|
||||
# Second iteration: Send snapshots
|
||||
def send_snapshots(fslist, snapshots, config, sudo):
|
||||
failed_snapshots = set()
|
||||
remote_hosts = {}
|
||||
remote_targets = {}
|
||||
log = logging.getLogger(LOGGER)
|
||||
for fs in fslist:
|
||||
conf = get_config_for_fs(fs, config)
|
||||
remote_snapshots = None
|
||||
@ -194,7 +177,6 @@ def main():
|
||||
except zsnaplib.ZFSSnapshotError as e:
|
||||
failed_snapshots.add(fs)
|
||||
log.warning(e)
|
||||
ret = RET_CODES['ERROR']
|
||||
continue
|
||||
remote_snapshots[remote_fs] = [base_snap]
|
||||
|
||||
@ -206,8 +188,7 @@ def main():
|
||||
break
|
||||
if not last_remote:
|
||||
failed_snapshots.add(fs)
|
||||
log.warning('No common snapshot local and remote, you need to create a new base copy!')
|
||||
ret = RET_CODES['ERROR']
|
||||
log.warning('{}: No common snapshot local and remote, you need to create a new base copy!'.format(fs))
|
||||
continue
|
||||
last_local = snapshots[fs][0]
|
||||
if last_remote == last_local:
|
||||
@ -229,8 +210,10 @@ def main():
|
||||
log.info('{} successfully sent to remote'.format(fs))
|
||||
except zsnaplib.ZFSSnapshotError as e:
|
||||
log.warning(e)
|
||||
return failed_snapshots
|
||||
|
||||
# Third iteration: weed old snapshots
|
||||
def weed_snapshots(fslist, snapshots, config, sudo, failed_snapshots):
|
||||
log = logging.getLogger(LOGGER)
|
||||
for fs in fslist:
|
||||
conf = get_config_for_fs(fs, config)
|
||||
if fs in failed_snapshots:
|
||||
@ -238,6 +221,8 @@ def main():
|
||||
continue
|
||||
if fs not in snapshots:
|
||||
continue
|
||||
if not conf['weed_enable']:
|
||||
continue
|
||||
|
||||
kwargs = {k: int(v) for k, v in conf.items() if k in [
|
||||
'keep_custom',
|
||||
@ -261,7 +246,60 @@ def main():
|
||||
**kwargs)
|
||||
|
||||
|
||||
def main():
|
||||
config = configparser.SafeConfigParser()
|
||||
config.read('/etc/zsnapper.ini')
|
||||
sudo = False
|
||||
ret = RET_CODES['SUCCESS']
|
||||
log = logging.getLogger(LOGGER)
|
||||
|
||||
if os.getuid() != 0:
|
||||
sudo = True
|
||||
|
||||
fslist = sorted(zsnaplib.get_filesystems(sudo))
|
||||
snapshots = zsnaplib.get_snapshots(sudo)
|
||||
|
||||
failed_snapshots = do_snapshots(fslist, snapshots, config, sudo)
|
||||
if failed_snapshots:
|
||||
ret = RET_CODES['ERROR']
|
||||
|
||||
lockfile = '/tmp/zsnapper.pid'
|
||||
# This loop should run at most twice
|
||||
while True:
|
||||
try:
|
||||
lockfd = os.open(lockfile, os.O_CREAT|os.O_EXCL|os.O_WRONLY, mode=0o640)
|
||||
os.write(lockfd, "{}".format(os.getpid()).encode('utf-8'))
|
||||
os.close(lockfd)
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# lock file exists, check if the pid seems valid
|
||||
with open(lockfile, 'r') as f:
|
||||
pid = f.read()
|
||||
try:
|
||||
pid = int(pid)
|
||||
os.kill(pid, 0)
|
||||
# If we got here the lock is owned by an existing pid
|
||||
log.info('Previous run is not completed yet, will not send or weed snapshots')
|
||||
return ret
|
||||
except OSError:
|
||||
# pid is not running, forcing unlock
|
||||
os.remove(lockfile)
|
||||
except ValueError:
|
||||
log.error('lockfile {} exists but does not seem to contain a pid. Will not continue'.format(lockfile))
|
||||
return RET_CODES['FAILED']
|
||||
|
||||
|
||||
# reload all snapshots so we get our new snapshots here
|
||||
snapshots = zsnaplib.get_snapshots(sudo)
|
||||
failed_send = send_snapshots(fslist, snapshots, config, sudo)
|
||||
if failed_send:
|
||||
ret = RET_CODES['ERROR']
|
||||
|
||||
failed_snapshots.update(failed_send)
|
||||
weed_snapshots(fslist, snapshots, config, sudo, failed_snapshots)
|
||||
os.remove(lockfile)
|
||||
|
||||
if __name__ == '__main__':
|
||||
log = logging.getLogger(LOGGER)
|
||||
@ -269,7 +307,10 @@ if __name__ == '__main__':
|
||||
handler = logging.StreamHandler()
|
||||
handler.setLevel(logging.WARNING)
|
||||
log.addHandler(handler)
|
||||
|
||||
handler = logging.handlers.SysLogHandler(address='/dev/log')
|
||||
formatter = logging.Formatter(fmt='zsnapper %(message)s')
|
||||
handler.setFormatter(formatter)
|
||||
handler.setLevel(logging.INFO)
|
||||
log.addHandler(handler)
|
||||
sys.exit(main())
|
||||
|
||||
@ -212,69 +212,69 @@ def weed_snapshots(
|
||||
keep['custom'].append(date)
|
||||
|
||||
if keep_yearly:
|
||||
saved['year'] = saved['year'][-keep_yearly:]
|
||||
keep['year'] = keep['year'][-keep_yearly:]
|
||||
else:
|
||||
saved['year'] = []
|
||||
keep['year'] = []
|
||||
|
||||
if keep_monthly:
|
||||
saved['month'] = saved['month'][-keep_monthly:]
|
||||
keep['month'] = keep['month'][-keep_monthly:]
|
||||
else:
|
||||
saved['month'] = []
|
||||
keep['month'] = []
|
||||
|
||||
if keep_weekly:
|
||||
saved['week'] = saved['week'][-keep_weekly:]
|
||||
keep['week'] = keep['week'][-keep_weekly:]
|
||||
else:
|
||||
saved['week'] = []
|
||||
keep['week'] = []
|
||||
|
||||
if keep_daily:
|
||||
saved['day'] = saved['day'][-keep_daily:]
|
||||
keep['day'] = keep['day'][-keep_daily:]
|
||||
else:
|
||||
saved['day'] = []
|
||||
keep['day'] = []
|
||||
|
||||
if keep_hourly:
|
||||
saved['hour'] = saved['hour'][-keep_hourly:]
|
||||
keep['hour'] = keep['hour'][-keep_hourly:]
|
||||
else:
|
||||
saved['hour'] = []
|
||||
keep['hour'] = []
|
||||
|
||||
if keep_30min:
|
||||
saved['min30'] = saved['min30'][-keep_30min:]
|
||||
keep['min30'] = keep['min30'][-keep_30min:]
|
||||
else:
|
||||
saved['min30'] = []
|
||||
keep['min30'] = []
|
||||
|
||||
if keep_15min:
|
||||
saved['min15'] = saved['min15'][-keep_15min:]
|
||||
keep['min15'] = keep['min15'][-keep_15min:]
|
||||
else:
|
||||
saved['min15'] = []
|
||||
keep['min15'] = []
|
||||
|
||||
if keep_5min:
|
||||
saved['min5'] = saved['min5'][-keep_5min:]
|
||||
keep['min5'] = keep['min5'][-keep_5min:]
|
||||
else:
|
||||
saved['min5'] = []
|
||||
keep['min5'] = []
|
||||
|
||||
if keep_1min:
|
||||
saved['min1'] = saved['min1'][-keep_1min:]
|
||||
keep['min1'] = keep['min1'][-keep_1min:]
|
||||
else:
|
||||
saved['min1'] = []
|
||||
keep['min1'] = []
|
||||
|
||||
if keep_custom:
|
||||
saved['custom'] = saved['custom'][-keep_custom:]
|
||||
keep['custom'] = keep['custom'][-keep_custom:]
|
||||
else:
|
||||
saved['custom'] = []
|
||||
keep['custom'] = []
|
||||
|
||||
all_saved = []
|
||||
all_saved.extend(saved['year'])
|
||||
all_saved.extend(saved['month'])
|
||||
all_saved.extend(saved['week'])
|
||||
all_saved.extend(saved['day'])
|
||||
all_saved.extend(saved['hour'])
|
||||
all_saved.extend(saved['min30'])
|
||||
all_saved.extend(saved['min15'])
|
||||
all_saved.extend(saved['min5'])
|
||||
all_saved.extend(saved['min1'])
|
||||
all_saved.extend(saved['custom'])
|
||||
all_saved = set(all_saved)
|
||||
all_keep = []
|
||||
all_keep.extend(keep['year'])
|
||||
all_keep.extend(keep['month'])
|
||||
all_keep.extend(keep['week'])
|
||||
all_keep.extend(keep['day'])
|
||||
all_keep.extend(keep['hour'])
|
||||
all_keep.extend(keep['min30'])
|
||||
all_keep.extend(keep['min15'])
|
||||
all_keep.extend(keep['min5'])
|
||||
all_keep.extend(keep['min1'])
|
||||
all_keep.extend(keep['custom'])
|
||||
all_keep = set(all_keep)
|
||||
|
||||
to_remove = [date for date in dates if date not in all_saved]
|
||||
to_remove = [date for date in dates if date not in all_keep]
|
||||
for date in to_remove:
|
||||
try:
|
||||
log.info('{}: removing snapshot from {}'.format(fs, date))
|
||||
|
||||
@ -1,33 +1,60 @@
|
||||
[settings]
|
||||
# NOTE:
|
||||
# --stdin is used to pass password to sudo, this does not work the first time
|
||||
# a user uses sudo, so make sure to run a sudo command manually first
|
||||
# zsnapper sample configuration
|
||||
|
||||
# Each section is the name of a ZFS file system
|
||||
# All settings are applied recursively to all file system descendants
|
||||
#
|
||||
# or preferably, leave this commented out and use NOPASSWD for sudo...
|
||||
;sudo=<password>
|
||||
|
||||
|
||||
[tank]
|
||||
# frequency of snapshots
|
||||
# Frequency of snapshots
|
||||
# Set to empty value to disable snapshoting
|
||||
#
|
||||
# Interval units are 'd', 'h' and 'm' for days, hours and minutes.
|
||||
# they can also be combined if you for example wants a snapshot taken every 1 and a half days:
|
||||
;snapshot_interval=1d 12h
|
||||
snapshot_interval=1h
|
||||
|
||||
# Remote replication
|
||||
# possible other value is 'latest' to only sync the last available snapshot
|
||||
# possible other value is 'latest' to only sync the latest snapshot
|
||||
# Set to empty value to not send the snapshots to remote
|
||||
remote_enable=all
|
||||
|
||||
# The remote_zfs_cmd option is the command to use to execute zfs on target machine.
|
||||
# remote_test_cmd, if set, is executed before trying to send any snapshot to remote.
|
||||
# If remote_test_cmd returns a non-zero status the remote is considered to be unavailable
|
||||
# and no snapshots are sent. (A warning is written in the log though)
|
||||
#
|
||||
# NOTE:
|
||||
# The command arguments must not contain whitespace characters, since
|
||||
# split() is used to create an array to subprocess.Popen()
|
||||
# The command arguments must not contain whitespace characters, due to implementation details.
|
||||
#
|
||||
# Variables can be used in remote_zfs_cmd and remote_test_cmd. Any setting
|
||||
# available in the section can be used as a variable
|
||||
remote_zfs_cmd=/usr/bin/ssh ${remote_user}@${remote_host} /usr/bin/sudo /sbin/zfs
|
||||
remote_test_cmd=/usr/bin/ssh ${remote_user}@${remote_host} echo "success"
|
||||
# The remote_host option is optional but recommended if you send snapshots to a remote host.
|
||||
remote_host=my.backup.server.tld
|
||||
# remote_user is not a actually a zsnapper option; but it's used as a variable in the remote commands.
|
||||
remote_user=backup
|
||||
remote_host=hem.winterbird.org
|
||||
remote_zfs_target=tank/backup/asuna/tank
|
||||
|
||||
# remote_zfs_target is the file system on the remote client that should receive zfs sends
|
||||
# for this file system.
|
||||
# NOTE:
|
||||
# Just like any other option this is inherited by file system descendants,
|
||||
# but if a child has the same remote_zfs_target as the parent, the child
|
||||
# will instead use this to figure out where the parent is and be sent to
|
||||
# it position relative to the parent.
|
||||
# For example: The local file system tank/ROOT will be sent to tank/backup/client/ROOT.
|
||||
remote_zfs_target=tank/backup/client
|
||||
|
||||
# These can be set to use custom arguments to zfs send and zfs receive
|
||||
; remote_send_flags=-D -p
|
||||
remote_send_flags=
|
||||
remote_send_flags=-D -p
|
||||
remote_recv_flags=
|
||||
|
||||
# snapshot weeding
|
||||
# set weed_enable to an empty value to disable snapshot weeding.
|
||||
# NOTE:
|
||||
# If weeding is enabled but no keep_<time> setting is configured all
|
||||
# your snapshots, except the latest, will be removed. Make sure to
|
||||
# configure your weeding settings carefully.
|
||||
weed_enable=1
|
||||
keep_hourly=24
|
||||
keep_daily=7
|
||||
keep_weekly=4
|
||||
@ -39,7 +66,8 @@ snapshot_interval=
|
||||
remote_enable=
|
||||
|
||||
[tank/media]
|
||||
snapshot_interval=15m
|
||||
snapshot_interval=
|
||||
remote_enable=
|
||||
|
||||
[tank/tmp]
|
||||
snapshot_interval=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user