first version of rotator
This commit is contained in:
parent
3814d8b07a
commit
863d7e77f8
GPG Key ID:
F4329687B0FA7F8D
204
rotator/__init__.py
Normal file
204
rotator/__init__.py
Normal file
@ -0,0 +1,204 @@
|
|||||||
|
import datetime
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import logging
|
||||||
|
|
||||||
|
_logger = 'rotator'
|
||||||
|
|
||||||
|
def parse_filename(name, regex, timeformat):
|
||||||
|
log = logging.getLogger(_logger)
|
||||||
|
match = regex.search(name)
|
||||||
|
if match:
|
||||||
|
prefix, date = match.groups()
|
||||||
|
log.debug('successfully parsed filename: {}'.format(name))
|
||||||
|
return (prefix, datetime.datetime.strptime(date, timeformat))
|
||||||
|
return (None, None)
|
||||||
|
|
||||||
|
def get_files_in_dir(path, regex, timeformat, recurse=False):
|
||||||
|
res = {}
|
||||||
|
for root,dirs,files in os.walk(path, topdown=False):
|
||||||
|
for f in files:
|
||||||
|
prefix, timestamp = parse_filename(f, regex, timeformat)
|
||||||
|
if prefix and prefix not in res:
|
||||||
|
res[prefix] = set()
|
||||||
|
res[prefix].add((os.path.join(root,f), timestamp))
|
||||||
|
if not recurse:
|
||||||
|
return res
|
||||||
|
return res
|
||||||
|
|
||||||
|
def rotate(
|
||||||
|
dirs,
|
||||||
|
regex,
|
||||||
|
timeformat,
|
||||||
|
recurse = False,
|
||||||
|
keep_yearly = 0,
|
||||||
|
keep_monthly = 0,
|
||||||
|
keep_weekly = 0,
|
||||||
|
keep_daily = 0,
|
||||||
|
keep_hourly = 0,
|
||||||
|
keep_30min = 0,
|
||||||
|
keep_15min = 0,
|
||||||
|
keep_5min = 0,
|
||||||
|
keep_1min = 0):
|
||||||
|
|
||||||
|
log = logging.getLogger(_logger)
|
||||||
|
|
||||||
|
merged_filelist = {}
|
||||||
|
for d in dirs:
|
||||||
|
files = get_files_in_dir(d, regex, timeformat, recurse)
|
||||||
|
for prefix in files.keys():
|
||||||
|
if prefix not in merged_filelist:
|
||||||
|
merged_filelist[prefix] = set()
|
||||||
|
merged_filelist[prefix].update(files[prefix])
|
||||||
|
|
||||||
|
for prefix, filelist in merged_filelist.items():
|
||||||
|
to_remove = get_deprecated_files(
|
||||||
|
filelist,
|
||||||
|
keep_yearly,
|
||||||
|
keep_monthly,
|
||||||
|
keep_weekly,
|
||||||
|
keep_daily,
|
||||||
|
keep_hourly,
|
||||||
|
keep_30min,
|
||||||
|
keep_15min,
|
||||||
|
keep_5min,
|
||||||
|
keep_1min)
|
||||||
|
for f in to_remove:
|
||||||
|
log.info("removing: {}".format(f))
|
||||||
|
os.remove(f)
|
||||||
|
|
||||||
|
def get_deprecated_files(
|
||||||
|
filelist,
|
||||||
|
keep_yearly,
|
||||||
|
keep_monthly,
|
||||||
|
keep_weekly,
|
||||||
|
keep_daily,
|
||||||
|
keep_hourly,
|
||||||
|
keep_30min,
|
||||||
|
keep_15min,
|
||||||
|
keep_5min,
|
||||||
|
keep_1min):
|
||||||
|
|
||||||
|
dates = [date for path,date in filelist]
|
||||||
|
|
||||||
|
keep = {
|
||||||
|
'year' : [],
|
||||||
|
'month' : [],
|
||||||
|
'week' : [],
|
||||||
|
'day' : [],
|
||||||
|
'hour' : [],
|
||||||
|
'min30' : [],
|
||||||
|
'min15' : [],
|
||||||
|
'min5' : [],
|
||||||
|
'min1' : []
|
||||||
|
}
|
||||||
|
saved = {
|
||||||
|
'year' : [],
|
||||||
|
'month' : [],
|
||||||
|
'week' : [],
|
||||||
|
'day' : [],
|
||||||
|
'hour' : [],
|
||||||
|
'min30' : [],
|
||||||
|
'min15' : [],
|
||||||
|
'min5' : [],
|
||||||
|
'min1' : []
|
||||||
|
}
|
||||||
|
|
||||||
|
for date in sorted(dates):
|
||||||
|
min1 = date-datetime.timedelta(seconds=date.second, microseconds=date.microsecond)
|
||||||
|
min5 = date-datetime.timedelta(minutes=date.minute%5, seconds=date.second, microseconds=date.microsecond)
|
||||||
|
min15 = date-datetime.timedelta(minutes=date.minute%15, seconds=date.second, microseconds=date.microsecond)
|
||||||
|
min30 = date-datetime.timedelta(minutes=date.minute%30, seconds=date.second, microseconds=date.microsecond)
|
||||||
|
hour = date-datetime.timedelta(minutes=date.minute, seconds=date.second, microseconds=date.microsecond)
|
||||||
|
day = datetime.datetime.combine(date.date(), datetime.time.min)
|
||||||
|
week = datetime.datetime.combine(date.date()-datetime.timedelta(days=date.weekday()), datetime.time.min)
|
||||||
|
month = datetime.datetime(year=date.year, month=date.month, day=1)
|
||||||
|
year = datetime.datetime(year=date.year, month=1, day=1)
|
||||||
|
|
||||||
|
if year not in saved['year']:
|
||||||
|
saved['year'].append(year)
|
||||||
|
keep['year'].append(date)
|
||||||
|
if month not in saved['month']:
|
||||||
|
saved['month'].append(month)
|
||||||
|
keep['month'].append(date)
|
||||||
|
if week not in saved['week']:
|
||||||
|
saved['week'].append(week)
|
||||||
|
keep['week'].append(date)
|
||||||
|
if day not in saved['day']:
|
||||||
|
saved['day'].append(day)
|
||||||
|
keep['day'].append(date)
|
||||||
|
if hour not in saved['hour']:
|
||||||
|
saved['hour'].append(hour)
|
||||||
|
keep['hour'].append(date)
|
||||||
|
if min30 not in saved['min30']:
|
||||||
|
saved['min30'].append(min30)
|
||||||
|
keep['min30'].append(date)
|
||||||
|
if min15 not in saved['min15']:
|
||||||
|
saved['min15'].append(min15)
|
||||||
|
keep['min15'].append(date)
|
||||||
|
if min5 not in saved['min5']:
|
||||||
|
saved['min5'].append(min5)
|
||||||
|
keep['min5'].append(date)
|
||||||
|
if min1 not in saved['min1']:
|
||||||
|
saved['min1'].append(min1)
|
||||||
|
keep['min1'].append(date)
|
||||||
|
|
||||||
|
if keep_yearly:
|
||||||
|
keep['year'] = keep['year'][-keep_yearly:]
|
||||||
|
else:
|
||||||
|
keep['year'] = []
|
||||||
|
|
||||||
|
if keep_monthly:
|
||||||
|
keep['month'] = keep['month'][-keep_monthly:]
|
||||||
|
else:
|
||||||
|
keep['month'] = []
|
||||||
|
|
||||||
|
if keep_weekly:
|
||||||
|
keep['week'] = keep['week'][-keep_weekly:]
|
||||||
|
else:
|
||||||
|
keep['week'] = []
|
||||||
|
|
||||||
|
if keep_daily:
|
||||||
|
keep['day'] = keep['day'][-keep_daily:]
|
||||||
|
else:
|
||||||
|
keep['day'] = []
|
||||||
|
|
||||||
|
if keep_hourly:
|
||||||
|
keep['hour'] = keep['hour'][-keep_hourly:]
|
||||||
|
else:
|
||||||
|
keep['hour'] = []
|
||||||
|
|
||||||
|
if keep_30min:
|
||||||
|
keep['min30'] = keep['min30'][-keep_30min:]
|
||||||
|
else:
|
||||||
|
keep['min30'] = []
|
||||||
|
|
||||||
|
if keep_15min:
|
||||||
|
keep['min15'] = keep['min15'][-keep_15min:]
|
||||||
|
else:
|
||||||
|
keep['min15'] = []
|
||||||
|
|
||||||
|
if keep_5min:
|
||||||
|
keep['min5'] = keep['min5'][-keep_5min:]
|
||||||
|
else:
|
||||||
|
keep['min5'] = []
|
||||||
|
|
||||||
|
if keep_1min:
|
||||||
|
keep['min1'] = keep['min1'][-keep_1min:]
|
||||||
|
else:
|
||||||
|
keep['min1'] = []
|
||||||
|
|
||||||
|
all_keep = []
|
||||||
|
all_keep.extend(keep['year'])
|
||||||
|
all_keep.extend(keep['month'])
|
||||||
|
all_keep.extend(keep['week'])
|
||||||
|
all_keep.extend(keep['day'])
|
||||||
|
all_keep.extend(keep['hour'])
|
||||||
|
all_keep.extend(keep['min30'])
|
||||||
|
all_keep.extend(keep['min15'])
|
||||||
|
all_keep.extend(keep['min5'])
|
||||||
|
all_keep.extend(keep['min1'])
|
||||||
|
all_keep = set(all_keep)
|
||||||
|
|
||||||
|
to_remove = [date for date in dates if date not in all_keep]
|
||||||
|
return [path for path,date in filelist if date not in all_keep]
|
||||||
120
rotator/__main__.py
Normal file
120
rotator/__main__.py
Normal file
@ -0,0 +1,120 @@
|
|||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import logging.handlers
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import rotator
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser(description="Rotate backups based on timestamp in names")
|
||||||
|
parser.add_argument(
|
||||||
|
'-y', '--yearly',
|
||||||
|
help='number of yearly backups to keep',
|
||||||
|
type=int,
|
||||||
|
default=0)
|
||||||
|
parser.add_argument(
|
||||||
|
'-m', '--monthly',
|
||||||
|
help='number of monthly backups to keep',
|
||||||
|
type=int,
|
||||||
|
default=0)
|
||||||
|
parser.add_argument(
|
||||||
|
'-w', '--weekly',
|
||||||
|
help='number of weekly backups to keep',
|
||||||
|
type=int,
|
||||||
|
default=0)
|
||||||
|
parser.add_argument(
|
||||||
|
'-d', '--daily',
|
||||||
|
help='number of daily backups to keep',
|
||||||
|
type=int,
|
||||||
|
default=0)
|
||||||
|
parser.add_argument(
|
||||||
|
'-o', '--hourly',
|
||||||
|
help='number of hourly backups to keep',
|
||||||
|
type=int,
|
||||||
|
default=0)
|
||||||
|
parser.add_argument(
|
||||||
|
'--min30',
|
||||||
|
help='number of half-hourly backups to keep',
|
||||||
|
type=int,
|
||||||
|
default=0)
|
||||||
|
parser.add_argument(
|
||||||
|
'--min15',
|
||||||
|
help='number of quarterly-hourly backups to keep',
|
||||||
|
type=int,
|
||||||
|
default=0)
|
||||||
|
parser.add_argument(
|
||||||
|
'--min5',
|
||||||
|
help='number of 5-minutely backups to keep',
|
||||||
|
type=int,
|
||||||
|
default=0)
|
||||||
|
parser.add_argument(
|
||||||
|
'--min1',
|
||||||
|
help='number of minutely backups to keep',
|
||||||
|
type=int,
|
||||||
|
default=0)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-r', '--regex',
|
||||||
|
help='''only rotate backups matching the given regex. Regex must
|
||||||
|
contain at least two match groups, the first being identifier for
|
||||||
|
the backup set and the second being the date.''',
|
||||||
|
default=r'^(.*)[-_.]([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{4})')
|
||||||
|
parser.add_argument(
|
||||||
|
'-t', '--time-format',
|
||||||
|
help='time format of the timestamp (python format)',
|
||||||
|
default='%Y-%m-%d_%H%M')
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-R', '--recurse',
|
||||||
|
action='store_true',
|
||||||
|
help='process paths recursively')
|
||||||
|
parser.add_argument(
|
||||||
|
'path',
|
||||||
|
help='Full path to the directory containing the backup files',
|
||||||
|
nargs='+')
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
def main():
|
||||||
|
log = logging.getLogger(rotator._logger)
|
||||||
|
log.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
handler = logging.StreamHandler()
|
||||||
|
handler.setLevel(logging.WARNING)
|
||||||
|
log.addHandler(handler)
|
||||||
|
|
||||||
|
handler = logging.handlers.SysLogHandler(address='/dev/log')
|
||||||
|
formatter = logging.Formatter(fmt='rotator[%(process)s] %(message)s')
|
||||||
|
handler.setFormatter(formatter)
|
||||||
|
handler.setLevel(logging.INFO)
|
||||||
|
log.addHandler(handler)
|
||||||
|
|
||||||
|
args = parse_args()
|
||||||
|
|
||||||
|
if all([not x for x in
|
||||||
|
[args.yearly,
|
||||||
|
args.monthly,
|
||||||
|
args.weekly,
|
||||||
|
args.daily,
|
||||||
|
args.hourly,
|
||||||
|
args.min30,
|
||||||
|
args.min15,
|
||||||
|
args.min5,
|
||||||
|
args.min1]]):
|
||||||
|
log.error('All time intervals set to 0, this would remove all backups, refusing to run')
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
rotator.rotate(
|
||||||
|
args.path,
|
||||||
|
re.compile(args.regex),
|
||||||
|
args.time_format,
|
||||||
|
recurse=args.recurse,
|
||||||
|
keep_yearly=args.yearly,
|
||||||
|
keep_monthly=args.monthly,
|
||||||
|
keep_weekly=args.weekly,
|
||||||
|
keep_daily=args.daily,
|
||||||
|
keep_hourly=args.hourly,
|
||||||
|
keep_30min=args.min30,
|
||||||
|
keep_15min=args.min15,
|
||||||
|
keep_5min=args.min5,
|
||||||
|
keep_1min=args.min1)
|
||||||
29
setup.py
Normal file
29
setup.py
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
import setuptools
|
||||||
|
|
||||||
|
with open('README.md', 'r') as fh:
|
||||||
|
long_description = fh.read()
|
||||||
|
|
||||||
|
setuptools.setup(
|
||||||
|
name='rotator',
|
||||||
|
version='0.1.0',
|
||||||
|
author='Fredrik Eriksson',
|
||||||
|
author_email='feffe@fulh.ax',
|
||||||
|
description='Simple script for rotating backups',
|
||||||
|
long_description=long_description,
|
||||||
|
long_description_content_type='text/markdown',
|
||||||
|
url='https://gitea.fulh.ax/feffe/rotator',
|
||||||
|
packages=setuptools.find_packages(),
|
||||||
|
license='BSD 3-caluse',
|
||||||
|
entry_points = {
|
||||||
|
'console_scripts': [
|
||||||
|
'rotator = rotator.__main__:main'
|
||||||
|
]
|
||||||
|
},
|
||||||
|
classifiers=[
|
||||||
|
'Development Status :: 3 - Alpha',
|
||||||
|
'Environment :: Console',
|
||||||
|
'Intended Audience :: System Administrators',
|
||||||
|
'License :: OSI Approved :: BSD License',
|
||||||
|
'Operating System :: POSIX',
|
||||||
|
],
|
||||||
|
)
|
||||||
Loading…
Reference in New Issue
Block a user