initial git import

This commit is contained in:
Fredrik Eriksson 2019-04-06 23:50:54 +02:00
parent ea733c2f55
commit 118f0b2d78
No known key found for this signature in database
GPG Key ID: 8825C73A0FD1502A
11 changed files with 747 additions and 0 deletions

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
build/
dist/
*.pyc
*.egg-info

57
bin/sau Executable file
View File

@ -0,0 +1,57 @@
#!/usr/bin/env python3.6
import configparser
import logging
import logging.handlers
import os
import platform
import sys
import sau
import sau.services
import sau.platforms
def main():
conf = sau.config
log = logging.getLogger(sau.LOGNAME)
reboot_required = False
platform = sau.platforms.get_platform()
#reboot_required = platform.system_upgrade()
platform.pkg_upgrade()
reboot_recommended = sau.services.restart_services()
if conf.getboolean('default', 'do_reboot', fallback=False):
if reboot_required or reboot_recommended:
os.system('/sbin/reboot')
if __name__ == '__main__':
sau.config = configparser.SafeConfigParser()
conf = sau.config
if platform.system() == 'FreeBSD':
syslog_socket = '/var/run/log'
conf_file = '/usr/local/etc/sau.cfg'
else:
syslog_socket = '/dev/log'
conf_file = '/etc/sau.cfg'
if os.path.isfile(conf_file):
conf.read(conf_file)
log = logging.getLogger(sau.LOGNAME)
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
formatter = logging.Formatter(fmt='%(asctime)s %(levelname)s: %(message)s')
handler.setFormatter(formatter)
handler.setLevel(logging.DEBUG)
log.addHandler(handler)
handler = logging.handlers.SysLogHandler(address=syslog_socket)
formatter = logging.Formatter(fmt='{}[%(process)s] %(message)s'.format(sau.LOGNAME))
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)
log.addHandler(handler)
sys.exit(main())

38
config.cfg Normal file
View File

@ -0,0 +1,38 @@
# Default settings in the default section (obviously)
[default]
# version_diff represents (hopefully) the compatibility change for the package.
# It works by starting with the number of dots in the version scheme + 1, and
# then removes 1 for every field that is the same in the current and new
# version. Some examples:
# 1.0.0 -> 1.0.0 (level 0, a reinstall of the same version)
# 1.0.0 -> 1.0.1 (level 1)
# 1.0.0 -> 1.1.0 (level 2)
# 1.0.0 -> 2.0.0 (level 3)
# 1.0.0.0 -> 2.0.0.0 (level 4)
# 1.0.1 -> 1.0.1.1 (level 2, because that's how it turned out...)
min_version_diff=2
# sau can reboot on system upgrades (FreeBSD) or if the service restarts does
# not close all deleted files (any platform)
do_reboot=no
# The packages section contains <package>=<version_diff> pairs to override the
# default min_version_diff. Note that package naming may differ depending on
# platform
[packages]
# Gentoo kernel stuff should be updated manually
sys-kernel/gentoo-sources=0
sys-kernel/spl=0
sys-fs/zfs-kmod=0
dev-db/postgresql=1
# FreeBSD uses the short package name
gitlab=1
# The services section contains <process-name>=<service-name>
# The process name is whatever psutil returns and I haven't checked if it's
# trustworthy, so it's probably not... Use with care.
[services]
gitlab-workhorse=gitlab

2
sau/__init__.py Normal file
View File

@ -0,0 +1,2 @@
LOGNAME="sau"

7
sau/errors.py Normal file
View File

@ -0,0 +1,7 @@
class PlatformNotSupported(Exception):
pass
class UnknownServiceError(Exception):
pass

177
sau/freebsd.py Normal file
View File

@ -0,0 +1,177 @@
import logging
import random
import re
import subprocess
import time
import sau
import sau.errors
import sau.helpers
PKG_PATH='/usr/sbin/pkg'
SERVICE_PATH='/usr/sbin/service'
FREEBSD_UPDATE_PATH='/usr/sbin/freebsd-update'
# regex to identify rc.d-files
rc_script_re = re.compile(r'^(?:/usr/local)?/etc/rc\.d/(.*)$')
# regex to parse packages
pkg_upgrade_re = re.compile(r'^\s([^\s]*): ([^\s]*) -> ([^\s]*).*$')
def identify_service_from_bin(exe):
log = logging.getLogger(sau.LOGNAME)
cmd = [ PKG_PATH, 'which', '-q', exe ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
pkg, err = proc.communicate()
if not pkg:
raise sau.errors.UnknownServiceError("'{}' does not belong to any package".format(exe))
pkg = pkg.decode('utf-8').strip()
log.debug('{} belongs to package {}'.format(exe, pkg))
cmd = [ PKG_PATH, 'info', '-l', pkg ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
rc_scripts = set()
for line in out.decode('utf-8').splitlines():
match = re.match(rc_script_re, line.strip())
if match:
log.debug('found potential rc-script for {} at {}'.format(exe, match.group(1)))
rc_scripts.add(match.group(1))
if len(rc_scripts) < 1:
raise sau.errors.UnknownServiceError("'{}' belongs to package '{}', but no rc-script was found".format(exe, pkg))
if len(rc_scripts) > 1:
raise sau.errors.UnkownServiceError("'{}' belongs to package '{}, but it contains multiple rc-scripts: {}".format(exe, pkg, ', '.join(rc_scripts)))
return rc_scripts.pop()
def restart_service(service):
log = logging.getLogger(sau.LOGNAME)
cmd = [ SERVICE_PATH, service, 'restart' ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0:
log.warning("Restart of {} failed:".format(service))
for line in out.splitlines():
log.warning("stdout: {}".format(line))
for line in err.splitlines():
log.warning("stderr: {}".format(line))
else:
log.info("restarted service {}".format(service))
def system_upgrade():
log = logging.getLogger(sau.LOGNAME)
# this is how freebsd wants it...
time.sleep(random.randint(0, 3600))
# now we can lie without soiling our conscience too much
cmd = [ FREEBSD_UPDATE_PATH, '--not-running-from-cron', 'fetch', 'install' ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
env = { 'PAGER': '/bin/cat' })
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode == 0:
if out.endswith('No updates are available to install.\n'):
log.debug('No system updates are available')
else:
log.info('System updates installed:')
for line in out.splitlines():
log.info(line)
return True
else:
log.warning('System upgrade failed (return code {}):'.format(proc.returncode))
for line in out.splitlines():
log.info('stdout: {}'.format(line))
for line in err.splitlines():
log.info('stderr: {}'.format(line))
return False
def pkg_upgrade():
log = logging.getLogger(sau.LOGNAME)
conf = sau.config
cmd = [ PKG_PATH, 'upgrade', '-nq' ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode == 0 and not out and not err:
log.debug('No package upgrades available')
return False
upgrades = []
for line in out.splitlines():
match = re.match(pkg_upgrade_re, line)
if match:
upgrades.append({
'pkg': match.group(1),
'version_old': match.group(2),
'version_new': match.group(3)
})
if not upgrades or err:
log.warning('Could not parse pkg output:')
for line in out.splitlines():
log.warning('stdout: {}'.format(line))
for line in err.splitlines():
log.warning('stderr: {}'.format(line))
return False
default_version_diff = conf.getint('default', 'min_version_diff', fallback=2)
for pkg in upgrades:
pkg['upgrade_level'] = sau.helpers.version_diff(pkg['version_new'], pkg['version_old'])
log.debug('pkg upgrade available {}'.format(pkg))
diff = conf.getint('packages', pkg['pkg'], fallback=default_version_diff)
if diff >= pkg['upgrade_level']:
log.debug('configured level {} >= pkg level {}'.format(diff, pkg['upgrade_level']))
pkg['upgrade'] = True
else:
log.debug('configured level {} < pkg level {}'.format(diff, pkg['upgrade_level']))
pkg['upgrade'] = False
upgradables = [x['pkg'] for x in upgrades if x['upgrade'] ]
if upgradables:
cmd = [ PKG_PATH, 'upgrade', '-yq' ] + upgradables
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0 or err:
log.warning('{} failed:'.format(' '.join(cmd)))
for line in out.splitlines():
log.warning('stdout: {}'.format(line))
for line in err.splitlines():
log.warning('stderr: {}'.format(line))
for pkg in [x for x in upgrades if not x['upgrade']]:
log.warning('Package require manual upgrade: {} {} -> {}'.format(pkg['pkg'], pkg['version_old'], pkg['version_new']))
return True

272
sau/gentoo.py Normal file
View File

@ -0,0 +1,272 @@
import logging
import re
import subprocess
import sau
import sau.helpers
EIX_SYNC_PATH='/usr/bin/eix-sync'
RC_SERVICE_PATH='/sbin/rc-service'
EMERGE_PATH='/usr/bin/emerge'
EQUERY_PATH='/usr/bin/equery'
# parsing output from eix -Ttnc
package_re = re.compile('^\[([^\]])\] ([^ ]*) \((.*)\): .*$')
# parsing version information from substrings of the above
slot_re = re.compile('^(\(~\))?([^\(]+)(\([^\)]+\))$')
def identify_service_from_bin(exe):
log = logging.getLogger(sau.LOGNAME)
init_script_re = re.compile(r'/etc/init\.d/(.*)')
cmd = [ EQUERY_PATH, '-Cq', 'b', exe ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0:
log.warning("searching for owner of {} failed:".format(exe))
for line in out.splitlines():
log.warning("stdout: {}".format(line))
for line in err.splitlines():
log.warning("stderr: {}".format(line))
return None
pkg = out.strip()
cmd = [ EQUERY_PATH, '-Cq', 'f', pkg ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0:
log.warning("listing files for package {} failed:".format(pkg))
for line in out.splitlines():
log.warning("stdout: {}".format(line))
for line in err.splitlines():
log.warning("stderr: {}".format(line))
return None
matches = set()
for line in out.splitlines():
match = re.match(init_script_re, line)
if match:
matches.add(match.group(1))
if len(matches) < 1:
log.warning('Could not find any init script in package {}'.format(pkg))
elif len(matches) > 1:
log.warning('Found multiple init script in package {}'.format(pkg))
else:
return matches.pop()
return None
def restart_service(service):
log = logging.getLogger(sau.LOGNAME)
cmd = [ SERVICE_PATH, service, 'restart' ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0:
log.warning("Restart of {} failed:".format(service))
for line in out.splitlines():
log.warning("stdout: {}".format(line))
for line in err.splitlines():
log.warning("stderr: {}".format(line))
else:
log.info("restarted service {}".format(service))
def system_upgrade():
log.debug('Gentoo has no concept of system upgrade, ignoring...')
return False
def _sync_portage():
log = logging.getLogger(sau.LOGNAME)
cmd = [ EMERGE_PATH, '-q', '--sync' ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0:
log.warning("Portage sync failed:")
for line in out.splitlines():
log.warning("stdout: {}".format(line))
for line in err.splitlines():
log.warning("stderr: {}".format(line))
def _parse_version(version):
slot_re = re.compile('^(\(~\))?([^\(]+)(\([^\)]+\))$')
match = re.match(slot_re, version)
return {
'keyword': match.group(1),
'version': match.group(2),
'slot': match.group(2)
}
def _get_upgradable_packages_eix():
log = logging.getLogger(sau.LOGNAME)
conf = sau.config
filter_re = re.compile('^(No |$|--$|The names of all|Found \d+)')
proc = subprocess.Popen(
[EIX, '-Ttnc'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
default_version_diff = conf.getint('default', 'min_version_diff', fallback=2)
upgradable = set()
for line in out.splitlines():
if re.match(filter_re, line):
continue
match = re.match(package_re, line)
if not match:
continue
if match.group(1) != 'U':
log.warning('Bad status for package {}, manual upgrade required'.format(match.group(2)))
continue
min_diff = conf.getint('packages', match.group(2), fallback=default_version_diff)
installed, latest = match.group(3).split(' -> ')
installed = installed.split()
latest = latest.split()
installed = [_parse_version(x) for x in installed]
latest = [_parse_version(x) for x in latest]
for pkg in installed:
latest_in_slot = [x for x in latest if x['slot'] == pkg['slot']]
if not latest_in_slot:
log.warning('Could not find latest version of {} in slot {}'.format(match.group(2), pkg['slot']))
continue
latest_in_slot = latest_in_slot[0]
if pkg['version'] == latest_in_slot['version']:
continue
diff = sau.helpers.version_diff(latest_in_slot['version'], pkg['version'])
if min_diff >= diff:
log.debug('configured level {} >= pkg level {}'.format(min_diff, diff))
upgrade_pkg.add('{}:{}'.format(match.group(2), pkg['slot']))
else:
log.debug('configured level {} < pkg level {}'.format(min_diff, diff))
return upgradable
def _get_upgradable_packages():
log = loggin.getLogger(sau.LOGNAME)
conf = sau.config
pretend_re = re.compile(r'^\[ebuild ([^\]]*)\] ([^ ]+)( \[([^\])]\])?')
def pkg_upgrade():
log = logging.getLogger(sau.LOGNAME)
conf = sau.config
_sync_portage()
#packages = _get_upgradable_packages()
# [ebuild U ] media-plugins/alsa-plugins-1.1.8 [1.1.6]
pretend_re = re.compile(r'^\[ebuild ([^\]]*)\] ([^ ]+)( \[[^\]]+\])?')
# media-plugins/alsa-plugins-1.1.8
version_re = re.compile(r'^(.*/.*)-(\d+.*)$')
ignore_re = re.compile(r'^(|.*caus.* rebuilds.*|.*scheduled for merge.*|.*waiting for lock on.*)$')
default_version_diff = conf.getint('default', 'min_version_diff', fallback=2)
cmd = [ EMERGE_PATH, '--color', 'n', '-uDNpq', '@world' ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if not proc.returncode == 0:
log.error('emerge pretend returned {}'.format(proc.returncode))
for line in out.splitlines():
log.error('stdout: {}'.format(line))
for line in err.splitlines():
log.error('stderr: {}'.format(line))
return False
do_rebuild = True
for line in out.splitlines():
if re.match(ignore_re, line):
continue
match = re.match(pretend_re, line)
if not match:
log.warning('No match on line: {}'.format(line))
continue
status = match.group(1)
name = match.group(2)
old = match.group(3)
if not old:
continue
old = old.strip(' []')
nmatch = re.match(version_re, name)
name = nmatch.group(1)
version = nmatch.group(2)
min_diff = conf.getint('packages', name, fallback=default_version_diff)
diff = sau.helpers.version_diff(version, old)
if min_diff >= diff:
log.debug('{}-{} -> {} configured level {} >= pkg level {}'.format(name, old, version, min_diff, diff))
else:
log.warning('{}-{} -> {} configured level {} < pkg level {}'.format(name, old, version, min_diff, diff))
do_rebuild = False
if not do_rebuild:
log.warning('Some packages require manual attention, did not upgrade')
return False
cmd = [ EMERGE_PATH, '--color', 'n', '-uDNq', '@world' ]
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if not proc.returncode == 0 or err:
log.error('emerge pretend returned {}'.format(proc.returncode))
for line in out.splitlines():
log.error('stdout: {}'.format(line))
for line in err.splitlines():
log.error('stderr: {}'.format(line))
else:
log.info('upgrade complete')
for line in out.splitlines():
log.warning('stdout: {}'.format(line))

17
sau/helpers.py Normal file
View File

@ -0,0 +1,17 @@
def version_diff(new, old):
""" This will return 100 if the versions are not properly compareable,
otherwise it will return the number of points that differ. Typically 3 is a
major upgrade, 2 minor, 1 point release and patch-levels. It can also
return 0 if both versions are the same."""
new_list = list(new.rsplit('.'))
old_list = list(old.rsplit('.'))
common = min(len(new_list), len(old_list))
longest = max(len(new_list), len(old_list))
diff = longest-common
for i in range(0, common):
if new_list[i] == old_list[i]:
continue
return len(new_list) - i + diff
return 0

20
sau/platforms.py Normal file
View File

@ -0,0 +1,20 @@
import platform
import sau.errors
import sau.freebsd
import sau.gentoo
def get_platform():
platform_mod = None
if platform.system() == 'FreeBSD':
platform_mod = sau.freebsd
elif platform.system() == 'Linux':
if 'gentoo' in platform.release():
platform_mod = sau.gentoo
if not platform_mod:
raise sau.errors.PlatformNotSupported("System: {} Release: {} Version: {} is not supported".format(
platform.system(),
platform.release(),
platform.version()))
return platform_mod

127
sau/services.py Normal file
View File

@ -0,0 +1,127 @@
#!/usr/bin/env python3.6
import logging
import os
import re
import subprocess
import time
import psutil
import sau
import sau.errors
import sau.platforms
def _get_deleted_open_files(proc):
files = set()
try:
for f in proc.open_files():
if f.path and os.path.exists(f.path):
continue
else:
files.add(f)
except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
pass
return files
def get_exe_file(name):
log = logging.getLogger(sau.LOGNAME)
search_paths = [
'/bin'
'/sbin'
'/usr/bin',
'/usr/sbin',
'/usr/local/bin',
'/usr/local/sbin',
'/libexec',
'/usr/libexec',
'/usr/local/libexec'
]
for path in search_paths:
if os.path.isdir(path):
for root, dirs, files in os.walk(path):
if name in files:
log.debug('Found binary for {} at {}'.format(name, root))
return os.path.join(root, name)
def restart_services():
log = logging.getLogger(sau.LOGNAME)
platform = sau.platforms.get_platform()
conf = sau.config
check_procs = set()
for proc in psutil.process_iter():
files = _get_deleted_open_files(proc)
if files:
log.debug('{} has open deleted files'.format(proc))
check_procs.add(proc)
# wait before the second test
time.sleep(1)
# perform a second check to remove potential false positives
service_procs = set()
retest_procs = set()
for proc in check_procs:
files = _get_deleted_open_files(proc)
if not files:
# no deleted open files for this process any longer
continue
try:
exe = proc.exe()
parents = proc.parents()
except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
# either of the above exceptions means the process has quit
continue
log.debug('will attempt to restart parent of {}'.format(proc))
if len(parents) < 2:
log.debug('{} is its own top parent'.format(proc))
service_procs.add(proc)
else:
log.debug('{} has top parent {}'.format(proc, parents[-2]))
service_procs.add(parents[-2])
retest_procs.add(proc)
services = set()
for proc in service_procs:
try:
service_exe = proc.exe()
proc_name = proc.name()
except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
log.debug('{} died before it could be restarted'.format(proc))
continue
service_name = conf.get('services', proc_name, fallback=None)
if not service_name:
# if the exe file has been deleted since started, service_exe will be empty
# and we'll have to guess
if not service_exe:
log.debug('Could not get full path to executable for process {}, will attempt to guess'.format(proc))
service_exe = get_exe_file(service_name)
if not service_exe:
log.error('Failed to find executable for process {}'.format(proc))
continue
try:
service_name = platform.identify_service_from_bin(service_exe)
except sau.errors.UnknownServiceError:
log.warning('Could not find service for process {}'.format(proc))
services.add(service_name)
for service in services:
platform.restart_service(service)
recommend_restart = False
for proc in retest_procs:
try:
exe = proc.exe()
name = proc.name()
except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
log.debug('{} was successfully killed'.format(proc))
continue
if _get_deleted_open_files(proc):
log.warning('{} still has deleted files open'.format(proc))
recommend_restart = True
return recommend_restart

26
setup.py Normal file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env python3.6
from os import environ
from setuptools import setup, find_packages
setup(
name='sau',
version='0.9.0',
description='Tool for auto-updating OS and packages',
author='Feffe',
author_email='feffe@fulh.ax',
url='https://fulh.ax/feffe/sau',
platforms=['FreeBSD', 'Linux'],
license='BSD',
packages=find_packages(),
classifiers=[
'Development Status :: 4 - Beta',
'Environment :: Console',
'Intended Audience :: System Administrators',
'Programming Language :: Python :: 3',
'Topic :: Utilities',
],
keywords='auto-update',
install_requires=['psutil>=5.6.1'],
scripts=['bin/sau']
)